diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000228648125Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000228648125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31745460c25afec0725d63468c22d638cee9e98b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000228648125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee059d9f5f9fc1ec69e69a94417369ced2df3ea7f84d841c21990f1427c145f +size 1613887 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2dfc3d957d72d622e79077bf8015ee87ff2b8054 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70442701824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92048031744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70442701824, + "active_bytes_peak": 92048031744, + "allocated_bytes_current": 70442701824, + "allocated_bytes_peak": 92048031744, + "device": 4, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_peak": 12227684864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95361695744, + "reserved_bytes_peak": 95361695744, + "segment_count_current": 1759, + "segment_count_peak": 1759, + "timestamp_utc": "2026-08-30T00:02:29.042643Z", + "world_size": 8 + }, + "allocated_bytes_current": 70442701824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92048031744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70442701824, + "active_bytes_peak": 92048031744, + "allocated_bytes_current": 70442701824, + "allocated_bytes_peak": 92048031744, + "device": 4, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_peak": 12227684864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95361695744, + "reserved_bytes_peak": 95361695744, + "segment_count_current": 1759, + "segment_count_peak": 1759, + "timestamp_utc": "2026-08-30T00:02:28.679933Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12227684864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95361695744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95361695744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1759, + "segment_count_current_delta": 0, + "segment_count_peak": 1759, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.073396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60c8ac1b8d521a08bc2ffd2fd73846d2399f429f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18b67e3e4d0a110f56f9419d222d408f0eabe6117189aeb0a474f3f6fb9f15a4 +size 1613889 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66df5062316e4c98bd863e036471797ee292c52f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70442701824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92048031744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70442701824, + "active_bytes_peak": 92048031744, + "allocated_bytes_current": 70442701824, + "allocated_bytes_peak": 92048031744, + "device": 4, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_peak": 12227684864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95361695744, + "reserved_bytes_peak": 95361695744, + "segment_count_current": 1759, + "segment_count_peak": 1759, + "timestamp_utc": "2026-08-30T00:02:29.438748Z", + "world_size": 8 + }, + "allocated_bytes_current": 70442701824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92048031744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70442701824, + "active_bytes_peak": 92048031744, + "allocated_bytes_current": 70442701824, + "allocated_bytes_peak": 92048031744, + "device": 4, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_peak": 12227684864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95361695744, + "reserved_bytes_peak": 95361695744, + "segment_count_current": 1759, + "segment_count_peak": 1759, + "timestamp_utc": "2026-08-30T00:02:29.076202Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1281993728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12227684864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95361695744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95361695744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1759, + "segment_count_current_delta": 0, + "segment_count_peak": 1759, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.469405Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2318b8a36e7d8b5e1577f4a1711c7d5dde58c1d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc1efc148313fb98501f4fa6409c287bbf447add003a1ee1180525cad94f93c5 +size 1613890 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f1e09676b168f8bef0697aa618c7ae45049366b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 47660868608, + "active_bytes_peak": 70438209536, + "active_bytes_peak_delta": 45747587072, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.472683Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 47660868608, + "allocated_bytes_peak": 70438209536, + "allocated_bytes_peak_delta": 45747587072, + "before": { + "active_bytes_current": 22777337856, + "active_bytes_peak": 24690622464, + "allocated_bytes_current": 22777337856, + "allocated_bytes_peak": 24690622464, + "device": 5, + "inactive_split_bytes_current": 572352512, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 33151778816, + "reserved_bytes_peak": 33151778816, + "segment_count_current": 971, + "segment_count_peak": 971, + "timestamp_utc": "2026-08-30T00:02:05.946861Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 714136576, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 11267644928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71844233216, + "reserved_bytes_current_delta": 38692454400, + "reserved_bytes_peak": 71844233216, + "reserved_bytes_peak_delta": 38692454400, + "segment_count_current": 1692, + "segment_count_current_delta": 721, + "segment_count_peak": 1692, + "segment_count_peak_delta": 721, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle", + "timestamp_utc": "2026-08-30T00:02:27.542532Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c79ee5c31aaaa7b29313508bfd829a3388c7ef7 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd97f46fe7797ddadb16a6e4bc798584d406c63291e1f349c59624ac68ed81d +size 3049530 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9377ea9dc0b20569cceb2d50ddf51e923cb620c0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92041671680, + "active_bytes_peak_delta": 21603462144, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.248783Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92041671680, + "allocated_bytes_peak_delta": 21603462144, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.550926Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 23609737216, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 23609737216, + "segment_count_current": 1746, + "segment_count_current_delta": 54, + "segment_count_peak": 1746, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.282574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..451a83599883c2943eea5dd19eb93bbf51ec007f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4b62bf0af7b015f64e1095dd7dff2e6ce762d12772b980d070b29d3e78bfdf +size 1501557 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ee6c999f04b183babb0a2c6c7968719725cb14f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 463360, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.647748Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 463360, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.285461Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.677017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4121c3c27eb7ee8b9d24c12e44f3cb3586f5ea36 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222d249ff9dca1114a959342aa5f75e22f26c3f23afc04fa3e659122e192dd6a +size 1559027 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..92346de3ade7c0da7f44bad7e86acb0966507b3a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.042629Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.679684Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.073007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36de0db7464686b3ff418e80a2121fc80b31db85 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50cd8fecc613203bfb2e5d6b6c8618a394ab018f22cdf9ed73f8e70f053a875c +size 1559121 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5983c4293bc5081fc830428329361576402d2fc6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.438765Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 5, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.075794Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.469294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b5696ad5e31e83b9c4c42d2e8bf75fdc8294ecb --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1aca06dbecca24b3b47c3fbfff09c4ed7cc9b5912ba1fd56338f3bb572615f8 +size 1558993 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e09f56c13df2213028e43654b036c04506ee50 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 47660868608, + "active_bytes_peak": 70438209536, + "active_bytes_peak_delta": 45747587072, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.472664Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 47660868608, + "allocated_bytes_peak": 70438209536, + "allocated_bytes_peak_delta": 45747587072, + "before": { + "active_bytes_current": 22777337856, + "active_bytes_peak": 24690622464, + "allocated_bytes_current": 22777337856, + "allocated_bytes_peak": 24690622464, + "device": 6, + "inactive_split_bytes_current": 572352512, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 33151778816, + "reserved_bytes_peak": 33151778816, + "segment_count_current": 971, + "segment_count_peak": 971, + "timestamp_utc": "2026-08-30T00:02:05.946932Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 714136576, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 11267644928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71844233216, + "reserved_bytes_current_delta": 38692454400, + "reserved_bytes_peak": 71844233216, + "reserved_bytes_peak_delta": 38692454400, + "segment_count_current": 1692, + "segment_count_current_delta": 721, + "segment_count_peak": 1692, + "segment_count_peak_delta": 721, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle", + "timestamp_utc": "2026-08-30T00:02:27.540528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a870577719c6e0a67d3b99d97ae0e621e3f9afa5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e55753131b4e2261bba94d80b2924d6650dc796b7700adff66c77c14c4cae8 +size 3049530 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..864666b11eb1b60a484d8d51b480817425d169bc --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92041671680, + "active_bytes_peak_delta": 21603462144, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.248775Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92041671680, + "allocated_bytes_peak_delta": 21603462144, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.548623Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 23609737216, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 23609737216, + "segment_count_current": 1746, + "segment_count_current_delta": 54, + "segment_count_peak": 1746, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.280749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01a1ed7137c4f785b0285f1c3dd31ad71a30deea --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efa26c1491f2d6c2946f387b0ee0760dbd35990fdf8bdbd05044093f07be4b6b +size 1501559 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4643f94e2a42fee623ddde13d91d1abcd2f25294 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 463360, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.647693Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 463360, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.283570Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.675437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2f4e5579f99f3a0b689b5e837b8237321f9e617 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d98287d71e523a7256595ec4409fa8ea1e389d69e510f64cc9f62e62fa581173 +size 1559029 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96173bb9fbcbcf0a0d7d879174e37f142435d2c2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.042625Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.678050Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.072404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa1dac2ece1fcc244a271b325b86daea9a50a228 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3141c1a980654643dabaf2fbf19a5f86d2d2efecd4e0190d5f5ef481d3cb2029 +size 1559123 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10be654d336c68608d2d5ed852fc28094a8181c5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.438735Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 6, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.075223Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.467539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..530336a37a46df16fa1e650b1784c0fce3d346ed --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b082901926d40f821fbc668237652c19fa42000b2f9f32a0044f1cd0bfaaf9dd +size 1558995 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ce3d2f9361f77bd5e52a3dda9379c33e9a8dbe4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 47660868608, + "active_bytes_peak": 70438209536, + "active_bytes_peak_delta": 45747587072, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.472668Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 47660868608, + "allocated_bytes_peak": 70438209536, + "allocated_bytes_peak_delta": 45747587072, + "before": { + "active_bytes_current": 22777337856, + "active_bytes_peak": 24690622464, + "allocated_bytes_current": 22777337856, + "allocated_bytes_peak": 24690622464, + "device": 7, + "inactive_split_bytes_current": 572352512, + "inactive_split_bytes_peak": 1025337344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 33151778816, + "reserved_bytes_peak": 33151778816, + "segment_count_current": 971, + "segment_count_peak": 971, + "timestamp_utc": "2026-08-30T00:02:05.946870Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 714136576, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 11267644928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71844233216, + "reserved_bytes_current_delta": 38692454400, + "reserved_bytes_peak": 71844233216, + "reserved_bytes_peak_delta": 38692454400, + "segment_count_current": 1692, + "segment_count_current_delta": 721, + "segment_count_peak": 1692, + "segment_count_peak_delta": 721, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle", + "timestamp_utc": "2026-08-30T00:02:27.540421Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58a374f370185663de96dc1c703a852fc054584c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1986a43bc8249b4db6cb6330f5e4db96409ba47e367ce58a3f2d1ac3be0698cb +size 3049528 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32d9992326b5e9bf9aa38b20480a8a5bbd41f42e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92041671680, + "active_bytes_peak_delta": 21603462144, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.248815Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92041671680, + "allocated_bytes_peak_delta": 21603462144, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 70438209536, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 70438209536, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71844233216, + "reserved_bytes_peak": 71844233216, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-30T00:02:27.548024Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 23609737216, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 23609737216, + "segment_count_current": 1746, + "segment_count_current_delta": 54, + "segment_count_peak": 1746, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.281430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bcde9f66a6e6801483a2fe2c5031536f7ae143a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32a1c028300f7f7a4a0557231c30edaaaf0b48a811ad25221458a9b5bf16b285 +size 1501559 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c20117bdf49b0ed35ab241c65cb6498927756772 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 463360, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.647750Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 463360, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92041671680, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92041671680, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.284151Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle", + "timestamp_utc": "2026-08-30T00:02:28.674999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28207b62ae124a2fb5ac6b3442a15dd654e6cb9f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f68728a88841660ed74a69057541e9d9336833dc5c428452b5c05082ba9eff +size 1559029 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78a4c4e816297c2d55cbd8dfa86d9a69c43622a8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.042673Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:28.677552Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.069963Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56288232854148601b1aec9f9f79d772cf2415f9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07e1709f057110e692b7c444caed0d17bb4617d5bb137e9f17f3cddc0f4cd7a +size 1559123 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3477ecb7df10f81850ae3f9deda32a1a06d58544 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 70438206464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 92042135040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.438777Z", + "world_size": 8 + }, + "allocated_bytes_current": 70438206464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 92042135040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 70438206464, + "active_bytes_peak": 92042135040, + "allocated_bytes_current": 70438206464, + "allocated_bytes_peak": 92042135040, + "device": 7, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_peak": 12292982272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95453970432, + "reserved_bytes_peak": 95453970432, + "segment_count_current": 1746, + "segment_count_peak": 1746, + "timestamp_utc": "2026-08-30T00:02:29.072580Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1286489088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12292982272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95453970432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 95453970432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1746, + "segment_count_current_delta": 0, + "segment_count_peak": 1746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle", + "timestamp_utc": "2026-08-30T00:02:29.467379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1db89bb6ded51456aac0b8e5a974a995fc2e4361 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73a8ba81c286e50a925efdafe625870ca009592535941a1680e19ee2259ae2b3 +size 1558995 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4cf81357acbeef7c59d707fd1a98c520650d29b5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_better_memory_dump_support +commit=51c387c7 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=0 +async_grad_reduce=1 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..00320e749977c381d858bc60b6f617c84e26562a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log @@ -0,0 +1,1302 @@ +[00:01:36] WARN: moonep_backend_async_grad_reduce_better_memory_dump_support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 51c387c7 +============================================================== +W0830 00:01:38.236000 271073 torch/distributed/run.py:766] +W0830 00:01:38.236000 271073 torch/distributed/run.py:766] ***************************************** +W0830 00:01:38.236000 271073 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 00:01:38.236000 271073 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to 0 is connected to 0 is connected to 0 peer ranks. 0 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 +0 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W830 00:01:56.441895105 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 00:01:56.442181308 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 00:01:56.442202971 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 00:01:56.443016796 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 00:01:56.447548869 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 00:01:56.447655991 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 00:01:56.448071498 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 00:01:56.608074872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.780 seconds +time to initialize megatron (seconds): 7.286 +[after megatron is initialized] datetime: 2026-08-30 00:02:02 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 00:02:05 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 00:02:05 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3397.41, 3407.14) + train/valid/test-data-iterators-setup ..........: (123.48, 126.00) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 00:02:05 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 544.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.52 MiB + total_size : 548.52 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5376, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [64, 74, 68, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, 99, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [29, 17, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-30 00:02:27] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 21524.9 | throughput per GPU (TFLOP/s/GPU): 8.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 67175.32958984375 | max allocated: 67175.330078125 | reserved: 68476.0 | max reserved: 68476.0 + [2026-08-30 00:02:28] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 776.8 | throughput per GPU (TFLOP/s/GPU): 232.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.336 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:02:28] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 398.9 | throughput per GPU (TFLOP/s/GPU): 453.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686371E+00 | loss scale: 1.0 | grad norm: 62.939 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:02:29] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 394.8 | throughput per GPU (TFLOP/s/GPU): 458.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338075E+01 | loss scale: 1.0 | grad norm: 20.301 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:02:29] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 396.0 | throughput per GPU (TFLOP/s/GPU): 456.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049910E+01 | loss scale: 1.0 | grad norm: 95.166 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 00:02:29 +[rank0]:[W830 00:02:30.006178917 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 00:02:31.929533701 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 00:02:31.099652422 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 00:02:31.115394431 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 00:02:31.141780136 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 00:02:32.425427924 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 00:02:32.626966226 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 00:02:32.092291203 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 00:02:38.142000 271073 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 00:02:38.143000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271263 closing signal SIGTERM +W0830 00:02:38.143000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271266 closing signal SIGTERM +W0830 00:02:38.144000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271269 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 271073 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..3af25daea6de30d6478f526ec8fce9b6974de5bf --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 821 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 46579 +1, 46573 +2, 46575 +3, 46573 +4, 46583 +5, 46579 +6, 46591 +7, 46579 +0, 47547 +1, 47549 +2, 47549 +3, 47549 +4, 47549 +5, 47549 +6, 47549 +7, 47549 +0, 48005 +1, 47851 +2, 48079 +3, 47845 +4, 47835 +5, 48065 +6, 47855 +7, 48073 +0, 48699 +1, 48349 +2, 48721 +3, 48355 +4, 48505 +5, 48715 +6, 48369 +7, 48743 +0, 49251 +1, 48907 +2, 49279 +3, 48905 +4, 49041 +5, 49279 +6, 48905 +7, 49279 +0, 49251 +1, 48907 +2, 49279 +3, 48907 +4, 49043 +5, 49279 +6, 48907 +7, 49279 +0, 58515 +1, 58601 +2, 58703 +3, 58539 +4, 58779 +5, 58703 +6, 58539 +7, 58703 +0, 83559 +1, 83563 +2, 83551 +3, 83543 +4, 83571 +5, 83553 +6, 83543 +7, 83553 +0, 83947 +1, 83951 +2, 83941 +3, 83931 +4, 83959 +5, 83941 +6, 83931 +7, 83941 +0, 84339 +1, 84343 +2, 84341 +3, 84323 +4, 84351 +5, 84341 +6, 84323 +7, 84341 +0, 88029 +1, 87985 +2, 87671 +3, 87773 +4, 87817 +5, 87931 +6, 87541 +7, 87831 +0, 95543 +1, 95455 +2, 95533 +3, 95309 +4, 95465 +5, 95551 +6, 95423 +7, 95579 +0, 116439 +1, 116343 +2, 116411 +3, 116193 +4, 116315 +5, 116411 +6, 116289 +7, 116411 +0, 116703 +1, 116611 +2, 116701 +3, 116459 +4, 116583 +5, 116701 +6, 116555 +7, 116701 +0, 116121 +1, 116611 +2, 116375 +3, 115877 +4, 116583 +5, 116439 +6, 116555 +7, 116375 +0, 114831 +1, 8500 +2, 114829 +3, 114587 +4, 115143 +5, 114829 +6, 115115 +7, 114829 +0, 1 +1, 3 +2, 196 +3, 114587 +4, 964 +5, 4 +6, 4 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0033e8122f9bb9faa915a51f8d91f2467c7d7c14 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69838439936, + "active_bytes_current_delta": 48030109696, + "active_bytes_peak": 69838443008, + "active_bytes_peak_delta": 46343805440, + "after": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 69838443008, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 69838443008, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71605157888, + "reserved_bytes_peak": 71605157888, + "segment_count_current": 2597, + "segment_count_peak": 2597, + "timestamp_utc": "2026-08-30T00:03:43.559140Z", + "world_size": 8 + }, + "allocated_bytes_current": 69838439936, + "allocated_bytes_current_delta": 48030109696, + "allocated_bytes_peak": 69838443008, + "allocated_bytes_peak_delta": 46343805440, + "before": { + "active_bytes_current": 21808330240, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21808330240, + "allocated_bytes_peak": 23494637568, + "device": 0, + "inactive_split_bytes_current": 1876904448, + "inactive_split_bytes_peak": 1876904448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25293750272, + "reserved_bytes_peak": 25293750272, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:03:20.789954Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_current_delta": -133255168, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 11374313472, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71605157888, + "reserved_bytes_current_delta": 46311407616, + "reserved_bytes_peak": 71605157888, + "reserved_bytes_peak_delta": 46311407616, + "segment_count_current": 2597, + "segment_count_current_delta": 1709, + "segment_count_peak": 2597, + "segment_count_peak_delta": 1709, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.800305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9c66964ca0ed7dfb10de442a17db2572bd33691 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d10d50059a6f4bcacf3f9ed0f5bbaed5db49b1129ca1c4b452da57c04966c9d5 +size 12137648 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5edbf379c1fbb0ed6b39a958a4d8ff52bffddbb4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69838439936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91028974592, + "active_bytes_peak_delta": 21190531584, + "after": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 91028974592, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 91028974592, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_peak": 93361012736, + "segment_count_current": 2767, + "segment_count_peak": 2767, + "timestamp_utc": "2026-08-30T00:03:44.599046Z", + "world_size": 8 + }, + "allocated_bytes_current": 69838439936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91028974592, + "allocated_bytes_peak_delta": 21190531584, + "before": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 69838443008, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 69838443008, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71605157888, + "reserved_bytes_peak": 71605157888, + "segment_count_current": 2597, + "segment_count_peak": 2597, + "timestamp_utc": "2026-08-30T00:03:43.836519Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_current_delta": 21755854848, + "reserved_bytes_peak": 93361012736, + "reserved_bytes_peak_delta": 21755854848, + "segment_count_current": 2767, + "segment_count_current_delta": 170, + "segment_count_peak": 2767, + "segment_count_peak_delta": 170, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.716971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6c1410fbbf0225ef6d89a3fa5134d6a865e9fc1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0898e3930797946afe6c920f1fe1a1cabfaad1b752d333a537b77bd66e6bb99d +size 4795970 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06eb312bd2cca22f922752127437c5a7c46eda52 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69838439936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91037882368, + "active_bytes_peak_delta": 8907776, + "after": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 91037882368, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 91037882368, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_peak": 93361012736, + "segment_count_current": 2767, + "segment_count_peak": 2767, + "timestamp_utc": "2026-08-30T00:03:45.380326Z", + "world_size": 8 + }, + "allocated_bytes_current": 69838439936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91037882368, + "allocated_bytes_peak_delta": 8907776, + "before": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 91028974592, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 91028974592, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_peak": 93361012736, + "segment_count_current": 2767, + "segment_count_peak": 2767, + "timestamp_utc": "2026-08-30T00:03:44.725760Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93361012736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2767, + "segment_count_current_delta": 0, + "segment_count_peak": 2767, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.486125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd580ab4d1d8c985c960c65bf2a746681fe9b9ad --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880339e9db0617788e75c2e22f2a0546c808b71b61f90175f39f760cd6a65596 +size 5312337 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e776f94aba7dd710e45835285620680bb0698d83 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69855483392, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91050207232, + "active_bytes_peak_delta": 12324864, + "after": { + "active_bytes_current": 69855483392, + "active_bytes_peak": 91050207232, + "allocated_bytes_current": 69855483392, + "allocated_bytes_peak": 91050207232, + "device": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_peak": 13251217920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_peak": 93562339328, + "segment_count_current": 2850, + "segment_count_peak": 2850, + "timestamp_utc": "2026-08-30T00:03:47.042627Z", + "world_size": 8 + }, + "allocated_bytes_current": 69855483392, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91050207232, + "allocated_bytes_peak_delta": 12324864, + "before": { + "active_bytes_current": 69838439936, + "active_bytes_peak": 91037882368, + "allocated_bytes_current": 69838439936, + "allocated_bytes_peak": 91037882368, + "device": 0, + "inactive_split_bytes_current": 1743649280, + "inactive_split_bytes_peak": 13251217920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93361012736, + "reserved_bytes_peak": 93361012736, + "segment_count_current": 2767, + "segment_count_peak": 2767, + "timestamp_utc": "2026-08-30T00:03:45.494095Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_current_delta": 201326592, + "reserved_bytes_peak": 93562339328, + "reserved_bytes_peak_delta": 201326592, + "segment_count_current": 2850, + "segment_count_current_delta": 83, + "segment_count_peak": 2850, + "segment_count_peak_delta": 83, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.152851Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e34840320f89b0d25d1d60495be8ebc2e2846d9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:549f39ba4b012ef7fb6672d576fd170f44c1b0f12300bfee9d0def15bd6c031a +size 5165025 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7937dfed4c02d1f4c6056efbf9b3f4eb44ca962 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69855483392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91050207232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69855483392, + "active_bytes_peak": 91050207232, + "allocated_bytes_current": 69855483392, + "allocated_bytes_peak": 91050207232, + "device": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_peak": 13251217920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_peak": 93562339328, + "segment_count_current": 2850, + "segment_count_peak": 2850, + "timestamp_utc": "2026-08-30T00:03:47.538017Z", + "world_size": 8 + }, + "allocated_bytes_current": 69855483392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91050207232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69855483392, + "active_bytes_peak": 91050207232, + "allocated_bytes_current": 69855483392, + "allocated_bytes_peak": 91050207232, + "device": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_peak": 13251217920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_peak": 93562339328, + "segment_count_current": 2850, + "segment_count_peak": 2850, + "timestamp_utc": "2026-08-30T00:03:47.161648Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93562339328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2850, + "segment_count_current_delta": 0, + "segment_count_peak": 2850, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.598252Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ff79f1576f4e59a704ad8cce3bb3936162eb151 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15b96966d439e291061881c345ef0e5ab7a4849a5e74c2fd0687fac31e423e1 +size 2603125 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc665737993885ddd93727b7f1e635fc5692b0d8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69855483392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91050207232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69855483392, + "active_bytes_peak": 91050207232, + "allocated_bytes_current": 69855483392, + "allocated_bytes_peak": 91050207232, + "device": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_peak": 13251217920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_peak": 93562339328, + "segment_count_current": 2850, + "segment_count_peak": 2850, + "timestamp_utc": "2026-08-30T00:03:47.973627Z", + "world_size": 8 + }, + "allocated_bytes_current": 69855483392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91050207232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69855483392, + "active_bytes_peak": 91050207232, + "allocated_bytes_current": 69855483392, + "allocated_bytes_peak": 91050207232, + "device": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_peak": 13251217920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_peak": 93562339328, + "segment_count_current": 2850, + "segment_count_peak": 2850, + "timestamp_utc": "2026-08-30T00:03:47.601021Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1936321024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13251217920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93562339328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93562339328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2850, + "segment_count_current_delta": 0, + "segment_count_peak": 2850, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.030675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1bf55d55c2ceeea44797f6b61a89ec5ee3d0f10f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ef1dc6c55dfd3556d6ddd5f3e528b1317e53fc7a14b37a2a47b6261a37960eb +size 2603125 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8312e5cc6796ab2385580d46ef9eaeb68b5317ba --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69837019648, + "active_bytes_current_delta": 48024862720, + "active_bytes_peak": 69837022720, + "active_bytes_peak_delta": 46342385152, + "after": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 69837022720, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 69837022720, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71508688896, + "reserved_bytes_peak": 71508688896, + "segment_count_current": 2587, + "segment_count_peak": 2587, + "timestamp_utc": "2026-08-30T00:03:43.559145Z", + "world_size": 8 + }, + "allocated_bytes_current": 69837019648, + "allocated_bytes_current_delta": 48024862720, + "allocated_bytes_peak": 69837022720, + "allocated_bytes_peak_delta": 46342385152, + "before": { + "active_bytes_current": 21812156928, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21812156928, + "allocated_bytes_peak": 23494637568, + "device": 1, + "inactive_split_bytes_current": 1984226816, + "inactive_split_bytes_peak": 1984226816, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:03:20.789984Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_current_delta": -335626240, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 11379507200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71508688896, + "reserved_bytes_current_delta": 46397390848, + "reserved_bytes_peak": 71508688896, + "reserved_bytes_peak_delta": 46397390848, + "segment_count_current": 2587, + "segment_count_current_delta": 1706, + "segment_count_peak": 2587, + "segment_count_peak_delta": 1706, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.798240Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71bb3ea655b3592af7be71ccc3d2db610d9a128a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000343559630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8854a28aced6e8373bf481b931c3cecc795acc46976e5b5936806d95d5a40e52 +size 10979886 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..808d3edc53120ae0268dcf8ff362a69b4cdf0925 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69837019648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91020434432, + "active_bytes_peak_delta": 21183411712, + "after": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 91020434432, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 91020434432, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93260349440, + "reserved_bytes_peak": 93260349440, + "segment_count_current": 2755, + "segment_count_peak": 2755, + "timestamp_utc": "2026-08-30T00:03:44.599056Z", + "world_size": 8 + }, + "allocated_bytes_current": 69837019648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91020434432, + "allocated_bytes_peak_delta": 21183411712, + "before": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 69837022720, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 69837022720, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71508688896, + "reserved_bytes_peak": 71508688896, + "segment_count_current": 2587, + "segment_count_peak": 2587, + "timestamp_utc": "2026-08-30T00:03:43.834948Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93260349440, + "reserved_bytes_current_delta": 21751660544, + "reserved_bytes_peak": 93260349440, + "reserved_bytes_peak_delta": 21751660544, + "segment_count_current": 2755, + "segment_count_current_delta": 168, + "segment_count_peak": 2755, + "segment_count_peak_delta": 168, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.702546Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2735072eb1392e091267c04e546233ad8ecb228a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000344599458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9828a3048b0099980cf9d457e89ec4da9ff04ad6920e86f635762594f96dd243 +size 4698208 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e6a4b15bbea55c46e4f185d7a18f77c91d740b6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69837019648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91038213632, + "active_bytes_peak_delta": 17779200, + "after": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 91038213632, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 91038213632, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93260349440, + "reserved_bytes_peak": 93260349440, + "segment_count_current": 2755, + "segment_count_peak": 2755, + "timestamp_utc": "2026-08-30T00:03:45.380330Z", + "world_size": 8 + }, + "allocated_bytes_current": 69837019648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91038213632, + "allocated_bytes_peak_delta": 17779200, + "before": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 91020434432, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 91020434432, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93260349440, + "reserved_bytes_peak": 93260349440, + "segment_count_current": 2755, + "segment_count_peak": 2755, + "timestamp_utc": "2026-08-30T00:03:44.710970Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93260349440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93260349440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2755, + "segment_count_current_delta": 0, + "segment_count_peak": 2755, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.480332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fdc422c2b68275573c154d9fbf554a7c67d7541 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000345380738Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d255829f51e52b681d7272fb1f96139472d56c0287ebc22531eb8a50f7238336 +size 4708936 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5167d79c286be4584b1deeba3cb4fe09c8932cc8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69854063104, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91048786944, + "active_bytes_peak_delta": 10573312, + "after": { + "active_bytes_current": 69854063104, + "active_bytes_peak": 91048786944, + "allocated_bytes_current": 69854063104, + "allocated_bytes_peak": 91048786944, + "device": 1, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_peak": 13363734016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93465870336, + "reserved_bytes_peak": 93465870336, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.042625Z", + "world_size": 8 + }, + "allocated_bytes_current": 69854063104, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91048786944, + "allocated_bytes_peak_delta": 10573312, + "before": { + "active_bytes_current": 69837019648, + "active_bytes_peak": 91038213632, + "allocated_bytes_current": 69837019648, + "allocated_bytes_peak": 91038213632, + "device": 1, + "inactive_split_bytes_current": 1648600576, + "inactive_split_bytes_peak": 13363734016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93260349440, + "reserved_bytes_peak": 93260349440, + "segment_count_current": 2755, + "segment_count_peak": 2755, + "timestamp_utc": "2026-08-30T00:03:45.488366Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93465870336, + "reserved_bytes_current_delta": 205520896, + "reserved_bytes_peak": 93465870336, + "reserved_bytes_peak_delta": 205520896, + "segment_count_current": 2840, + "segment_count_current_delta": 85, + "segment_count_peak": 2840, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.151363Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30730820153d57a3b5c6e51824e3dfd2996185f3 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000347043095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4076a5527c61b53c21e7935d9c6c6204e97200d856096906df35cd38450adf5d +size 4824683 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6109a48f22cac1510258977cb624e07a6e4b2b4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69854063104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91048786944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69854063104, + "active_bytes_peak": 91048786944, + "allocated_bytes_current": 69854063104, + "allocated_bytes_peak": 91048786944, + "device": 1, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_peak": 13363734016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93465870336, + "reserved_bytes_peak": 93465870336, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.538017Z", + "world_size": 8 + }, + "allocated_bytes_current": 69854063104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91048786944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69854063104, + "active_bytes_peak": 91048786944, + "allocated_bytes_current": 69854063104, + "allocated_bytes_peak": 91048786944, + "device": 1, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_peak": 13363734016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93465870336, + "reserved_bytes_peak": 93465870336, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.160178Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93465870336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93465870336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.597530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ea5ea14e2a02e867f7990b90842602c9585e25b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000347538435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeb92927706763afea3de4d5a86b109bc11fb769f8048d02466e16fe91465c7b +size 2346948 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61104a9c11a212dae312601385d553d0a9b05d66 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69854063104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91048786944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69854063104, + "active_bytes_peak": 91048786944, + "allocated_bytes_current": 69854063104, + "allocated_bytes_peak": 91048786944, + "device": 1, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_peak": 13363734016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93465870336, + "reserved_bytes_peak": 93465870336, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.973607Z", + "world_size": 8 + }, + "allocated_bytes_current": 69854063104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91048786944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69854063104, + "active_bytes_peak": 91048786944, + "allocated_bytes_current": 69854063104, + "allocated_bytes_peak": 91048786944, + "device": 1, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_peak": 13363734016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93465870336, + "reserved_bytes_peak": 93465870336, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.600289Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1841272320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13363734016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93465870336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93465870336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.029894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f9844a7919f9af21abbc5d9627fdf7465ac9533 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260830T000347974000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a17a28c5573e1533702c33df98141aef2faf29b906bd3f1e0c7dc3964c4d38 +size 2346979 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac46d6218b5641a8548a7b77cb531ed2478c4bb0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 48024529408, + "active_bytes_peak": 69834803712, + "active_bytes_peak_delta": 46340166144, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.559194Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 48024529408, + "allocated_bytes_peak": 69834803712, + "allocated_bytes_peak_delta": 46340166144, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 2, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:03:20.789913Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": -119286272, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 11793103872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71598866432, + "reserved_bytes_current_delta": 46275756032, + "reserved_bytes_peak": 71598866432, + "reserved_bytes_peak_delta": 46275756032, + "segment_count_current": 2585, + "segment_count_current_delta": 1697, + "segment_count_peak": 2585, + "segment_count_peak_delta": 1697, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.808702Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb3926bc41a5262a8a2b4af0db836b9df9cc625c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000343559676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c39a46c179fa20b06b10209254e79ca57921aaadb0cb35ec071c535b0461c29 +size 12124848 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5087f50355e77a223b9477437f441f5cd95abffb --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91020424192, + "active_bytes_peak_delta": 21185620480, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.599137Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91020424192, + "allocated_bytes_peak_delta": 21185620480, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.848791Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 21732786176, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 21732786176, + "segment_count_current": 2744, + "segment_count_current_delta": 159, + "segment_count_peak": 2744, + "segment_count_peak_delta": 159, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.704810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ac08c58fe481e4dec4327d8e9d019da3448c939 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000344599557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4248cf83936ffbbc476672f0f1d4d1713f4b17bc98ef5481d1cfb93ea1e30c02 +size 4766521 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2437acf0dc00d3fc25bcb7ae30bc6953a0df7278 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91035160576, + "active_bytes_peak_delta": 14736384, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.380415Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91035160576, + "allocated_bytes_peak_delta": 14736384, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.713935Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2744, + "segment_count_current_delta": 0, + "segment_count_peak": 2744, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.485928Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2839f66759d2e28f3aefac6b8582afe1b4cd17a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000345380821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa4ea7e00f592e8c59e5bf0d8b4d2ee99afc63cd773fc3caad82fad3b55d7a22 +size 5302749 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d2d7e0280584f0fc24bc18f8bf0192255b07b1b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 11407360, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 2, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.042705Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 11407360, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 2, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.494944Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 228589568, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 228589568, + "segment_count_current": 2840, + "segment_count_current_delta": 96, + "segment_count_peak": 2840, + "segment_count_peak_delta": 96, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.154952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbbadc3b2c8e8b20573b4c7cfdbaef91f3c3811f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000347043202Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdad86d40936dccfb464b9249cf15f040af40bfae11b4398c2f19c5eb433490a +size 5156208 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a92c542f18f2ccd1044f192b0f1a20f6208396f4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 2, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.538111Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 2, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.164868Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.601139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9857b97fde2ffd051200c34bf4247b5c0ffc0e4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000347538552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b579ac9bc329d4def60da7ee1d9aee0a545e1b0a521d190f764c5a0d872bedb +size 2594984 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45e30e420e56b250263b8fdf3920f7c61e71e6c8 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 2, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.973693Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 2, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.603937Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.035136Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd44a34e5cb27a7e25c3648597019b5915c9d501 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260830T000347974099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ce6b62d4dca23d09926bdaba6b75bd2dba73561d143da1b102bd23941cef11 +size 2594984 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ef2d4d7f69b52fd03b47361674341dda17e780 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69833597440, + "active_bytes_current_delta": 48017208832, + "active_bytes_peak": 69833600512, + "active_bytes_peak_delta": 46338962944, + "after": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 69833600512, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 69833600512, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71349305344, + "reserved_bytes_peak": 71349305344, + "segment_count_current": 2575, + "segment_count_peak": 2575, + "timestamp_utc": "2026-08-30T00:03:43.559091Z", + "world_size": 8 + }, + "allocated_bytes_current": 69833597440, + "allocated_bytes_current_delta": 48017208832, + "allocated_bytes_peak": 69833600512, + "allocated_bytes_peak_delta": 46338962944, + "before": { + "active_bytes_current": 21816388608, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21816388608, + "allocated_bytes_peak": 23494637568, + "device": 3, + "inactive_split_bytes_current": 1919177728, + "inactive_split_bytes_peak": 1919177728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:03:20.789931Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_current_delta": -426538496, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 11815163392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71349305344, + "reserved_bytes_current_delta": 46238007296, + "reserved_bytes_peak": 71349305344, + "reserved_bytes_peak_delta": 46238007296, + "segment_count_current": 2575, + "segment_count_current_delta": 1694, + "segment_count_peak": 2575, + "segment_count_peak_delta": 1694, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.813585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2993b308fdb67130ae567fca38b4126355e9fc5a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000343559611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a8894c2d51ade662bf8baff8c70a3bab7465611360984ce50831234c41144f7 +size 14462519 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eabd55e6e91c192c76d9660f3c086598667aecfc --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69833597440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91024439808, + "active_bytes_peak_delta": 21190839296, + "after": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 91024439808, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 91024439808, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93103063040, + "reserved_bytes_peak": 93103063040, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.599037Z", + "world_size": 8 + }, + "allocated_bytes_current": 69833597440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91024439808, + "allocated_bytes_peak_delta": 21190839296, + "before": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 69833600512, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 69833600512, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71349305344, + "reserved_bytes_peak": 71349305344, + "segment_count_current": 2575, + "segment_count_peak": 2575, + "timestamp_utc": "2026-08-30T00:03:43.852928Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93103063040, + "reserved_bytes_current_delta": 21753757696, + "reserved_bytes_peak": 93103063040, + "reserved_bytes_peak_delta": 21753757696, + "segment_count_current": 2744, + "segment_count_current_delta": 169, + "segment_count_peak": 2744, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.705004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5fcb1cf82a5a649d217ae5b2651d544bf02c8e0d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000344599455Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1ac764db65302ce95ae71a7a285ee873f043cb630718f28d525dedee7cce8a +size 4898096 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60b264938b0b7ada1d61f4c9e4af78b0c96048db --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69833597440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91034116096, + "active_bytes_peak_delta": 9676288, + "after": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 91034116096, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 91034116096, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93103063040, + "reserved_bytes_peak": 93103063040, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.380298Z", + "world_size": 8 + }, + "allocated_bytes_current": 69833597440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91034116096, + "allocated_bytes_peak_delta": 9676288, + "before": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 91024439808, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 91024439808, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93103063040, + "reserved_bytes_peak": 93103063040, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.714103Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93103063040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93103063040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2744, + "segment_count_current_delta": 0, + "segment_count_peak": 2744, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.484387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2dee031c5df19652a5a87712b551f6ec272ff725 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000345380722Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117d60f55fdf43b619c43d4198320e3d377258f2967660abe5d45ad6219b5346 +size 5433965 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa8d52c85f7d8a4f97987f9cdb2e2c0132c396f7 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69850640896, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91045364736, + "active_bytes_peak_delta": 11248640, + "after": { + "active_bytes_current": 69850640896, + "active_bytes_peak": 91045364736, + "allocated_bytes_current": 69850640896, + "allocated_bytes_peak": 91045364736, + "device": 3, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_peak": 13734341120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93306486784, + "reserved_bytes_peak": 93306486784, + "segment_count_current": 2828, + "segment_count_peak": 2828, + "timestamp_utc": "2026-08-30T00:03:47.042657Z", + "world_size": 8 + }, + "allocated_bytes_current": 69850640896, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91045364736, + "allocated_bytes_peak_delta": 11248640, + "before": { + "active_bytes_current": 69833597440, + "active_bytes_peak": 91034116096, + "allocated_bytes_current": 69833597440, + "allocated_bytes_peak": 91034116096, + "device": 3, + "inactive_split_bytes_current": 1492639232, + "inactive_split_bytes_peak": 13734341120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93103063040, + "reserved_bytes_peak": 93103063040, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.493206Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93306486784, + "reserved_bytes_current_delta": 203423744, + "reserved_bytes_peak": 93306486784, + "reserved_bytes_peak_delta": 203423744, + "segment_count_current": 2828, + "segment_count_current_delta": 84, + "segment_count_peak": 2828, + "segment_count_peak_delta": 84, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.155358Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de80cff56da77d376fd74a799032419b68590654 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000347043180Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d176ce7ce95fdbccc269967d6a4f2317e8a76144d6272a7831e9c76cc2eb92 +size 5043112 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54094e4c9b7cdf9f4840a9b2c565f0a7caddb1d2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69850640896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91045364736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69850640896, + "active_bytes_peak": 91045364736, + "allocated_bytes_current": 69850640896, + "allocated_bytes_peak": 91045364736, + "device": 3, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_peak": 13734341120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93306486784, + "reserved_bytes_peak": 93306486784, + "segment_count_current": 2828, + "segment_count_peak": 2828, + "timestamp_utc": "2026-08-30T00:03:47.538087Z", + "world_size": 8 + }, + "allocated_bytes_current": 69850640896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91045364736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69850640896, + "active_bytes_peak": 91045364736, + "allocated_bytes_current": 69850640896, + "allocated_bytes_peak": 91045364736, + "device": 3, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_peak": 13734341120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93306486784, + "reserved_bytes_peak": 93306486784, + "segment_count_current": 2828, + "segment_count_peak": 2828, + "timestamp_utc": "2026-08-30T00:03:47.165265Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93306486784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93306486784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2828, + "segment_count_current_delta": 0, + "segment_count_peak": 2828, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.606140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d38763543309549334ff58dbdaf0dd3336bcb78d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000347538515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd647f798b7149c291208ee5922914c291a8b90fe92c49e1b9326b4f6b5c4752 +size 2554170 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a34a416b10fd1ffdc16a73919821e1aaadee7e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69850640896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91045364736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69850640896, + "active_bytes_peak": 91045364736, + "allocated_bytes_current": 69850640896, + "allocated_bytes_peak": 91045364736, + "device": 3, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_peak": 13734341120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93306486784, + "reserved_bytes_peak": 93306486784, + "segment_count_current": 2828, + "segment_count_peak": 2828, + "timestamp_utc": "2026-08-30T00:03:47.973620Z", + "world_size": 8 + }, + "allocated_bytes_current": 69850640896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91045364736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69850640896, + "active_bytes_peak": 91045364736, + "allocated_bytes_current": 69850640896, + "allocated_bytes_peak": 91045364736, + "device": 3, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_peak": 13734341120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93306486784, + "reserved_bytes_peak": 93306486784, + "segment_count_current": 2828, + "segment_count_peak": 2828, + "timestamp_utc": "2026-08-30T00:03:47.609003Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1685310976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13734341120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93306486784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93306486784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2828, + "segment_count_current_delta": 0, + "segment_count_peak": 2828, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.035153Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56166c4c12645b0333691a3b93b6906b5304e895 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260830T000347974030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83c96bc7b3cbec91e3545512291d522a7d244af04cc35ae0821f54879d80e86 +size 2554170 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f18cbb6d67a1d69615a88596fa1146620f4a5f7e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844616704, + "active_bytes_current_delta": 48036640768, + "active_bytes_peak": 69844619776, + "active_bytes_peak_delta": 46349982208, + "after": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 69844619776, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 69844619776, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2582, + "segment_count_peak": 2582, + "timestamp_utc": "2026-08-30T00:03:43.559108Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844616704, + "allocated_bytes_current_delta": 48036640768, + "allocated_bytes_peak": 69844619776, + "allocated_bytes_peak_delta": 46349982208, + "before": { + "active_bytes_current": 21807975936, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21807975936, + "allocated_bytes_peak": 23494637568, + "device": 4, + "inactive_split_bytes_current": 1965339136, + "inactive_split_bytes_peak": 1965339136, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:03:20.789922Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_current_delta": -353695744, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 11166027776, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71479328768, + "reserved_bytes_current_delta": 46368030720, + "reserved_bytes_peak": 71479328768, + "reserved_bytes_peak_delta": 46368030720, + "segment_count_current": 2582, + "segment_count_current_delta": 1701, + "segment_count_peak": 2582, + "segment_count_peak_delta": 1701, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.806456Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1e6fdcd08d0b701cda2654bc3441b59763f2831 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000343559627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831b322b9c039dac6e88da7f3701fee0c7f7a24d603ec3a8b4d8397ac1518de6 +size 14464560 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30385c94097894fe0fe4bbdfa4a1e2cbb18d3b65 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844616704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91023567872, + "active_bytes_peak_delta": 21178948096, + "after": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 91023567872, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 91023567872, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93230989312, + "reserved_bytes_peak": 93230989312, + "segment_count_current": 2750, + "segment_count_peak": 2750, + "timestamp_utc": "2026-08-30T00:03:44.599020Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844616704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91023567872, + "allocated_bytes_peak_delta": 21178948096, + "before": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 69844619776, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 69844619776, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2582, + "segment_count_peak": 2582, + "timestamp_utc": "2026-08-30T00:03:43.844193Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93230989312, + "reserved_bytes_current_delta": 21751660544, + "reserved_bytes_peak": 93230989312, + "reserved_bytes_peak_delta": 21751660544, + "segment_count_current": 2750, + "segment_count_current_delta": 168, + "segment_count_peak": 2750, + "segment_count_peak_delta": 168, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.704031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a483e2e9acfd8f606e6f7e69642f1e587be91de --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000344599433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e89d937ed20d7590c97ccd02722fd0224828f72cff24c0d7c4b105e125ee8ac5 +size 4797276 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a0fddd55c4cffd1e721d0abc24f47eaad3f40ca --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844616704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91044539392, + "active_bytes_peak_delta": 20971520, + "after": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 91044539392, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 91044539392, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93230989312, + "reserved_bytes_peak": 93230989312, + "segment_count_current": 2750, + "segment_count_peak": 2750, + "timestamp_utc": "2026-08-30T00:03:45.380294Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844616704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91044539392, + "allocated_bytes_peak_delta": 20971520, + "before": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 91023567872, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 91023567872, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93230989312, + "reserved_bytes_peak": 93230989312, + "segment_count_current": 2750, + "segment_count_peak": 2750, + "timestamp_utc": "2026-08-30T00:03:44.712350Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93230989312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93230989312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2750, + "segment_count_current_delta": 0, + "segment_count_peak": 2750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.484368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae1615134ed6f63f021d3770944a07e02157a67b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000345380720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f2452636e365b3980a16d411ff7e76ed9c6db7eb15e390708c94fa66733f0b3 +size 5313932 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3a9f26f5dbecb9f7a1101063676165da2a4cb4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69861660160, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91056384000, + "active_bytes_peak_delta": 11844608, + "after": { + "active_bytes_current": 69861660160, + "active_bytes_peak": 91056384000, + "allocated_bytes_current": 69861660160, + "allocated_bytes_peak": 91056384000, + "device": 4, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_peak": 13131366912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 2835, + "segment_count_peak": 2835, + "timestamp_utc": "2026-08-30T00:03:47.042615Z", + "world_size": 8 + }, + "allocated_bytes_current": 69861660160, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91056384000, + "allocated_bytes_peak_delta": 11844608, + "before": { + "active_bytes_current": 69844616704, + "active_bytes_peak": 91044539392, + "allocated_bytes_current": 69844616704, + "allocated_bytes_peak": 91044539392, + "device": 4, + "inactive_split_bytes_current": 1611643392, + "inactive_split_bytes_peak": 13131366912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93230989312, + "reserved_bytes_peak": 93230989312, + "segment_count_current": 2750, + "segment_count_peak": 2750, + "timestamp_utc": "2026-08-30T00:03:45.492443Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93436510208, + "reserved_bytes_current_delta": 205520896, + "reserved_bytes_peak": 93436510208, + "reserved_bytes_peak_delta": 205520896, + "segment_count_current": 2835, + "segment_count_current_delta": 85, + "segment_count_peak": 2835, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.151368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..040573901d3bf7a6f0c23101ce7b8ff6d6890eac --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000347043101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d30e79e8c519abeb7cb70eeeda7ea680af5d595cb520ae4e45fb6fcf333b717 +size 5205161 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ef214fc85a1fd078f7c8f5190689adb6acb6873 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69861660160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91056384000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69861660160, + "active_bytes_peak": 91056384000, + "allocated_bytes_current": 69861660160, + "allocated_bytes_peak": 91056384000, + "device": 4, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_peak": 13131366912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 2835, + "segment_count_peak": 2835, + "timestamp_utc": "2026-08-30T00:03:47.538021Z", + "world_size": 8 + }, + "allocated_bytes_current": 69861660160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91056384000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69861660160, + "active_bytes_peak": 91056384000, + "allocated_bytes_current": 69861660160, + "allocated_bytes_peak": 91056384000, + "device": 4, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_peak": 13131366912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 2835, + "segment_count_peak": 2835, + "timestamp_utc": "2026-08-30T00:03:47.160357Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93436510208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93436510208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2835, + "segment_count_current_delta": 0, + "segment_count_peak": 2835, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.601090Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bde9925fdb12b34a6d67d48527e833b64708a22 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000347538451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bde05b4fb245efbaf09354916d2b12dae6722f950018103f04aa3ac365d829b +size 2642873 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59350062eb3310f978449fd3590eaeaf56cf89a9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69861660160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91056384000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69861660160, + "active_bytes_peak": 91056384000, + "allocated_bytes_current": 69861660160, + "allocated_bytes_peak": 91056384000, + "device": 4, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_peak": 13131366912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 2835, + "segment_count_peak": 2835, + "timestamp_utc": "2026-08-30T00:03:47.973606Z", + "world_size": 8 + }, + "allocated_bytes_current": 69861660160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91056384000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69861660160, + "active_bytes_peak": 91056384000, + "allocated_bytes_current": 69861660160, + "allocated_bytes_peak": 91056384000, + "device": 4, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_peak": 13131366912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 2835, + "segment_count_peak": 2835, + "timestamp_utc": "2026-08-30T00:03:47.603869Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1804315136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13131366912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93436510208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93436510208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2835, + "segment_count_current_delta": 0, + "segment_count_peak": 2835, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.035581Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74435c225c30e35784d6b0bf20210047a01bec15 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260830T000347974008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1b06742d154ef5c834b8901976032bc937396357cb7075ed0f76b900e00ae5 +size 2642904 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..829c1c66055096e98589e4399ae873ec3fc17d4f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 48024529408, + "active_bytes_peak": 69834803712, + "active_bytes_peak_delta": 46340166144, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.559124Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 48024529408, + "allocated_bytes_peak": 69834803712, + "allocated_bytes_peak_delta": 46340166144, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 5, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:03:20.789919Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": -119286272, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 11793103872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71598866432, + "reserved_bytes_current_delta": 46275756032, + "reserved_bytes_peak": 71598866432, + "reserved_bytes_peak_delta": 46275756032, + "segment_count_current": 2585, + "segment_count_current_delta": 1697, + "segment_count_peak": 2585, + "segment_count_peak_delta": 1697, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.804408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e569c52210aaf75c3256fe59591c0dbacd8cf8ed --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000343559631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0abf99f26cc0f8008864a2663af330bff292932b806c7476105f898132515c6 +size 12124851 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f523208e2f3e03ee23b6e7f7567cabf30a0ef9f2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91020424192, + "active_bytes_peak_delta": 21185620480, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.599013Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91020424192, + "allocated_bytes_peak_delta": 21185620480, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.835514Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 21732786176, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 21732786176, + "segment_count_current": 2744, + "segment_count_current_delta": 159, + "segment_count_peak": 2744, + "segment_count_peak_delta": 159, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.697118Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c42a046e139464393cba6818137fb75fd4abebc2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000344599424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c279f884ef4aa1229246656af2828612ae756881fb97391d5dbaaa5f4ef2a3ae +size 4766522 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09f5bcc2e83d512f49583ba80e2efad829572b51 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91035160576, + "active_bytes_peak_delta": 14736384, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.380280Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91035160576, + "allocated_bytes_peak_delta": 14736384, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.704660Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2744, + "segment_count_current_delta": 0, + "segment_count_peak": 2744, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.473113Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7933202e46620fbba6df7ac0f29825f77eb81986 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000345380672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f9865406330290f449d3cd831790608d0c344496d63fb30ad4328499f189f2 +size 5302751 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60534ed3c4a6804c9669a73a824c2c3ceebeb032 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 11407360, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 5, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.042571Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 11407360, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 5, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.480302Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 228589568, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 228589568, + "segment_count_current": 2840, + "segment_count_current_delta": 96, + "segment_count_peak": 2840, + "segment_count_peak_delta": 96, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.147626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..738548c142c06ac1a2bd97976a973b861a1a0ca0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000347042999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec1f3faada9f5b4c61b042d45ca23375912cdc1cb2bbbcb1cc876176ee51d4c +size 5156208 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df24b4227c5f1de2daa51f44737b1fca6687a6e4 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 5, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.537982Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 5, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.155686Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.591550Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e78dea0610a82b09ad354735b75a9a7b3edf6a5f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000347538382Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2f3bdcd0e8bdabc81cd90491789fb0e82cca45c92ae04c4c8cf325324e9a6f +size 2594984 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fba7c69220fb2ba18b4850a11815d8ec0d746241 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 5, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.973590Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 5, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.594276Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.025677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f6dcdae60da9ea692bbdc0bfa0a662d236050ab --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260830T000347973978Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6975459d2c64f7308c805627d83860aecf943f347278d92b60fde1b985fb31e0 +size 2594984 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a94f223dde4b1ff530df69d5603fea79cf346ee --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69827939840, + "active_bytes_current_delta": 48015782912, + "active_bytes_peak": 69827942912, + "active_bytes_peak_delta": 46333305344, + "after": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 69827942912, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 69827942912, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71449968640, + "reserved_bytes_peak": 71449968640, + "segment_count_current": 2583, + "segment_count_peak": 2583, + "timestamp_utc": "2026-08-30T00:03:43.559111Z", + "world_size": 8 + }, + "allocated_bytes_current": 69827939840, + "allocated_bytes_current_delta": 48015782912, + "allocated_bytes_peak": 69827942912, + "allocated_bytes_peak_delta": 46333305344, + "before": { + "active_bytes_current": 21812156928, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21812156928, + "allocated_bytes_peak": 23494637568, + "device": 6, + "inactive_split_bytes_current": 1986323968, + "inactive_split_bytes_peak": 1986323968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:03:20.789920Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_current_delta": -387363840, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 11622842368, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71449968640, + "reserved_bytes_current_delta": 46338670592, + "reserved_bytes_peak": 71449968640, + "reserved_bytes_peak_delta": 46338670592, + "segment_count_current": 2583, + "segment_count_current_delta": 1702, + "segment_count_peak": 2583, + "segment_count_peak_delta": 1702, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.807206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11c064418ef0888f107fd03e833118d340afdb32 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000343559630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7872963d2c542b61f262dddd61d611f9b952103969544dbe3a33a43ee34536b4 +size 10980114 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b27ec2d1292d55234f660f59e044463cda13c5e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69827939840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91008265728, + "active_bytes_peak_delta": 21180322816, + "after": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 91008265728, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 91008265728, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93203726336, + "reserved_bytes_peak": 93203726336, + "segment_count_current": 2752, + "segment_count_peak": 2752, + "timestamp_utc": "2026-08-30T00:03:44.599063Z", + "world_size": 8 + }, + "allocated_bytes_current": 69827939840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91008265728, + "allocated_bytes_peak_delta": 21180322816, + "before": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 69827942912, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 69827942912, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71449968640, + "reserved_bytes_peak": 71449968640, + "segment_count_current": 2583, + "segment_count_peak": 2583, + "timestamp_utc": "2026-08-30T00:03:43.846321Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93203726336, + "reserved_bytes_current_delta": 21753757696, + "reserved_bytes_peak": 93203726336, + "reserved_bytes_peak_delta": 21753757696, + "segment_count_current": 2752, + "segment_count_current_delta": 169, + "segment_count_peak": 2752, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.706726Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..82f375c736e68c38bc56aa1c32ac2080be3ade6f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000344599479Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f42bc6edcdf561cd6fc457b2c2bd798eec7406057c20440c62e846f4c0a9518 +size 4698866 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.json new file mode 100644 index 0000000000000000000000000000000000000000..513210ddc9ff343cbc31e7a43921eb6f326a0674 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69827939840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91028712960, + "active_bytes_peak_delta": 20447232, + "after": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 91028712960, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 91028712960, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93203726336, + "reserved_bytes_peak": 93203726336, + "segment_count_current": 2752, + "segment_count_peak": 2752, + "timestamp_utc": "2026-08-30T00:03:45.380329Z", + "world_size": 8 + }, + "allocated_bytes_current": 69827939840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91028712960, + "allocated_bytes_peak_delta": 20447232, + "before": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 91008265728, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 91008265728, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93203726336, + "reserved_bytes_peak": 93203726336, + "segment_count_current": 2752, + "segment_count_peak": 2752, + "timestamp_utc": "2026-08-30T00:03:44.715839Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93203726336, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93203726336, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2752, + "segment_count_current_delta": 0, + "segment_count_peak": 2752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.486726Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ac7056d9196551bb661eaad8ad5db08875a5757 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000345380754Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73ce2bf6ce2014f430999735b0602a292e545eac9116f251b5f2b578a3b2022f +size 4709337 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef31488d60c18f3ab4cfa2e6ec31ddc94413121a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844983296, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91039707136, + "active_bytes_peak_delta": 10994176, + "after": { + "active_bytes_current": 69844983296, + "active_bytes_peak": 91039707136, + "allocated_bytes_current": 69844983296, + "allocated_bytes_peak": 91039707136, + "device": 6, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_peak": 13609166336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93407150080, + "reserved_bytes_peak": 93407150080, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-30T00:03:47.042609Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844983296, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91039707136, + "allocated_bytes_peak_delta": 10994176, + "before": { + "active_bytes_current": 69827939840, + "active_bytes_peak": 91028712960, + "allocated_bytes_current": 69827939840, + "allocated_bytes_peak": 91028712960, + "device": 6, + "inactive_split_bytes_current": 1598960128, + "inactive_split_bytes_peak": 13609166336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93203726336, + "reserved_bytes_peak": 93203726336, + "segment_count_current": 2752, + "segment_count_peak": 2752, + "timestamp_utc": "2026-08-30T00:03:45.495636Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93407150080, + "reserved_bytes_current_delta": 203423744, + "reserved_bytes_peak": 93407150080, + "reserved_bytes_peak_delta": 203423744, + "segment_count_current": 2836, + "segment_count_current_delta": 84, + "segment_count_peak": 2836, + "segment_count_peak_delta": 84, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.154720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f4d4a508af03d8c9b8c86f74004044d2035732c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000347043083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e3eff970c5196d442efde8edf49782c807a4fb429cee7e5482f27a0bde4af2 +size 4805504 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa21eac8605539c1e3584c5eb30d3e958254c834 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844983296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039707136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69844983296, + "active_bytes_peak": 91039707136, + "allocated_bytes_current": 69844983296, + "allocated_bytes_peak": 91039707136, + "device": 6, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_peak": 13609166336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93407150080, + "reserved_bytes_peak": 93407150080, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-30T00:03:47.538022Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844983296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039707136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69844983296, + "active_bytes_peak": 91039707136, + "allocated_bytes_current": 69844983296, + "allocated_bytes_peak": 91039707136, + "device": 6, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_peak": 13609166336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93407150080, + "reserved_bytes_peak": 93407150080, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-30T00:03:47.164591Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93407150080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93407150080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2836, + "segment_count_current_delta": 0, + "segment_count_peak": 2836, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.601274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e0f7b13ec36cce90758944d90e248e8918bf812 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000347538432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34012b16f5c6d3218b6fe5b9eab36f8648030c52e5f8655c1a35688de9a4f0b4 +size 2328808 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..488e83bad623d4e18f49b9b964d92b64a49095db --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69844983296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039707136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69844983296, + "active_bytes_peak": 91039707136, + "allocated_bytes_current": 69844983296, + "allocated_bytes_peak": 91039707136, + "device": 6, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_peak": 13609166336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93407150080, + "reserved_bytes_peak": 93407150080, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-30T00:03:47.973603Z", + "world_size": 8 + }, + "allocated_bytes_current": 69844983296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039707136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69844983296, + "active_bytes_peak": 91039707136, + "allocated_bytes_current": 69844983296, + "allocated_bytes_peak": 91039707136, + "device": 6, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_peak": 13609166336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93407150080, + "reserved_bytes_peak": 93407150080, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-30T00:03:47.604029Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1791631872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13609166336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93407150080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93407150080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2836, + "segment_count_current_delta": 0, + "segment_count_peak": 2836, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.034211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58749ff02490b69abd4396e644cf120aa486b2b6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260830T000347973994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dafdcd33c3da49ebc60dd0d57de2140efb3127b74ca7dfc992626ca078c78d2 +size 2328808 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7572c4cee8694c5b92278b0ce02617c81f2390e0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 48024529408, + "active_bytes_peak": 69834803712, + "active_bytes_peak_delta": 46340166144, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.559223Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 48024529408, + "allocated_bytes_peak": 69834803712, + "allocated_bytes_peak_delta": 46340166144, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 7, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:03:20.789833Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": -119286272, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 11793103872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71598866432, + "reserved_bytes_current_delta": 46275756032, + "reserved_bytes_peak": 71598866432, + "reserved_bytes_peak_delta": 46275756032, + "segment_count_current": 2585, + "segment_count_current_delta": 1697, + "segment_count_peak": 2585, + "segment_count_peak_delta": 1697, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.pickle", + "timestamp_utc": "2026-08-30T00:03:43.805429Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5cb1a8fe3f8471a4a2cde1657e7f8da3f0362ac --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000343559741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765a5503eaccc42be7c18a606374fa05fde0c88c7bcf20480a9bb761ccc11b8b +size 12124849 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8903e9082cd7f9de02df235287f05fe8ddbfb556 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91020424192, + "active_bytes_peak_delta": 21185620480, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.599059Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91020424192, + "allocated_bytes_peak_delta": 21185620480, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 69834803712, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 69834803712, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71598866432, + "reserved_bytes_peak": 71598866432, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:03:43.843391Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 21732786176, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 21732786176, + "segment_count_current": 2744, + "segment_count_current_delta": 159, + "segment_count_peak": 2744, + "segment_count_peak_delta": 159, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.pickle", + "timestamp_utc": "2026-08-30T00:03:44.704361Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6023b907876c969829b911748096290e9fe7ad72 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000344599481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bed92f6df59bb09f0ec0d12882ed95206ddd37a5fc4514b0b59dfd0fa24660c +size 4766520 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a1ab5907926ceba28785c9088adaba5cef31ec3 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69834800640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91035160576, + "active_bytes_peak_delta": 14736384, + "after": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.380330Z", + "world_size": 8 + }, + "allocated_bytes_current": 69834800640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91035160576, + "allocated_bytes_peak_delta": 14736384, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91020424192, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91020424192, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:44.712652Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93331652608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93331652608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2744, + "segment_count_current_delta": 0, + "segment_count_peak": 2744, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.pickle", + "timestamp_utc": "2026-08-30T00:03:45.483806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af4344f9b80014d13d052199e6c516d147f525e2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000345380739Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b2b98aa473e57833e8095e6474e05d100d8f84a8944c81fad1ba5412babe76f +size 5302743 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bced4c115a6310657f3c1ab6ffe5213732ec9ecd --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 11407360, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 7, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.042677Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 11407360, + "before": { + "active_bytes_current": 69834800640, + "active_bytes_peak": 91035160576, + "allocated_bytes_current": 69834800640, + "allocated_bytes_peak": 91035160576, + "device": 7, + "inactive_split_bytes_current": 1745191424, + "inactive_split_bytes_peak": 13657581568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93331652608, + "reserved_bytes_peak": 93331652608, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-30T00:03:45.491958Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 192671744, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 228589568, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 228589568, + "segment_count_current": 2840, + "segment_count_current_delta": 96, + "segment_count_peak": 2840, + "segment_count_peak_delta": 96, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.158168Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..086e9ece409f2dbe0d367d71dc904ca1ec8d230c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000347043161Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c8661c10ad30f521ee8d9dc367e67263b701b01d63ad877e085c25be378ea6 +size 5156201 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9256f87e8c4c64a0c81277251765b1b3354a6110 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 7, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.538061Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 7, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.167203Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.pickle", + "timestamp_utc": "2026-08-30T00:03:47.601873Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e8533f1a2c5f9a2df7c3f3e2c926fc0a9163f37 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000347538500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13b72149860e93b38da7dde88a480d76a83d7778d39a3ab634ca700cde0217f +size 2594977 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ce6d2b4ea1cb312c4710388d37fd9a87638451f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69851844096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91046567936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 7, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.973640Z", + "world_size": 8 + }, + "allocated_bytes_current": 69851844096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91046567936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69851844096, + "active_bytes_peak": 91046567936, + "allocated_bytes_current": 69851844096, + "allocated_bytes_peak": 91046567936, + "device": 7, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_peak": 13657581568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93560242176, + "reserved_bytes_peak": 93560242176, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-30T00:03:47.604633Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1937863168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13657581568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93560242176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93560242176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.pickle", + "timestamp_utc": "2026-08-30T00:03:48.035244Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..774abe1db4ca5ed25f1fd94ad69c8297c1775524 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260830T000347974048Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cf2ef92f4ae46f3b58bca3b5ee87b21b3a7e8dfa1ce79caa147774193e4542a +size 2594977 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..13b007f9718f1ca9220cc47f780a828d74311fa3 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_better_memory_dump_support +commit=51c387c7 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=1 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8517e8d7ec61d43dd920af86f4dfa343b24eb580 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/train.log @@ -0,0 +1,2729 @@ +[00:02:50] WARN: moonep_backend_async_grad_reduce_better_memory_dump_support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 51c387c7 +============================================================== +W0830 00:02:51.379000 273656 torch/distributed/run.py:766] +W0830 00:02:51.379000 273656 torch/distributed/run.py:766] ***************************************** +W0830 00:02:51.379000 273656 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 00:02:51.379000 273656 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 47 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 30 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7[Gloo] Rank 7 +1 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 456 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W830 00:03:09.241258016 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 00:03:09.241525686 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 00:03:09.241675074 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 00:03:09.241677140 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 00:03:09.243539840 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 00:03:09.243547448 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 00:03:09.245252474 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 00:03:09.406269562 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.979 seconds +time to initialize megatron (seconds): 7.800 +[after megatron is initialized] datetime: 2026-08-30 00:03:15 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.self_attention.linear_proj.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.43.self_attention.linear_qkv.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.28.self_attention.linear_qkv.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.26.self_attention.linear_proj.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.mlp.router.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.16.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.q_layernorm.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.1.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 00:03:20 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 00:03:20 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5231.48, 5234.42) + train/valid/test-data-iterators-setup ..........: (121.69, 126.45) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 00:03:20 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 144.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 147.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [0, 2048, 1408, 1536, 3968, 2944, 4224, 896, 2816, 1152, 2432, 2176, 896, 2560, 1152, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [110, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [71, 77, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +Number of parameters in transformer block in billions: 29.90 [2026-08-30 00:03:43] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 22769.3 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.354 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66603.12646484375 | max allocated: 66603.126953125 | reserved: 68288.0 | max reserved: 68288.0 + [2026-08-30 00:03:44] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1040.3 | throughput per GPU (TFLOP/s/GPU): 85.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.500 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:03:45] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 781.2 | throughput per GPU (TFLOP/s/GPU): 114.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008747E+01 | loss scale: 1.0 | grad norm: 180.908 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-30 00:03:47] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1662.4 | throughput per GPU (TFLOP/s/GPU): 53.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604850E+00 | loss scale: 1.0 | grad norm: 8.726 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:03:47] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 495.4 | throughput per GPU (TFLOP/s/GPU): 179.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307376E+00 | loss scale: 1.0 | grad norm: 4.339 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:03:47] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 435.3 | throughput per GPU (TFLOP/s/GPU): 204.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.264483E+00 | loss scale: 1.0 | grad norm: 6.416 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 00:03:48 +[rank0]:[W830 00:03:49.602278496 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 00:03:51.432992968 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 00:03:51.445545085 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 00:03:51.453315953 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 00:03:51.365000 273656 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 00:03:51.366000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273842 closing signal SIGTERM +W0830 00:03:51.367000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273843 closing signal SIGTERM +W0830 00:03:51.368000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273844 closing signal SIGTERM +W0830 00:03:51.369000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273845 closing signal SIGTERM +W0830 00:03:51.370000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273846 closing signal SIGTERM +W0830 00:03:51.371000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273847 closing signal SIGTERM +W0830 00:03:51.371000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273848 closing signal SIGTERM +W0830 00:03:51.372000 273656 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 273849 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 273656 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..9fe41a9bdab5a40703d2917dc3b2251c94271f92 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 29 +4, 629 +5, 4 +6, 629 +7, 10 +0, 923 +1, 911 +2, 931 +3, 921 +4, 923 +5, 931 +6, 925 +7, 927 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 16121 +1, 16833 +2, 16593 +3, 16833 +4, 16633 +5, 16593 +6, 16873 +7, 16577 +0, 47207 +1, 47207 +2, 47207 +3, 47207 +4, 47207 +5, 47207 +6, 47207 +7, 47207 +0, 47549 +1, 47551 +2, 47551 +3, 47551 +4, 47551 +5, 47551 +6, 47551 +7, 47551 +0, 48115 +1, 47941 +2, 48143 +3, 47941 +4, 47941 +5, 48143 +6, 47941 +7, 48143 +0, 48651 +1, 48285 +2, 48667 +3, 48279 +4, 48433 +5, 48697 +6, 48309 +7, 48689 +0, 49231 +1, 48889 +2, 49261 +3, 48887 +4, 49025 +5, 49259 +6, 48887 +7, 49259 +0, 49233 +1, 48889 +2, 49261 +3, 48889 +4, 49025 +5, 49261 +6, 48889 +7, 49261 +0, 58413 +1, 58377 +2, 58391 +3, 58393 +4, 58421 +5, 58391 +6, 58377 +7, 58391 +0, 83453 +1, 83467 +2, 83479 +3, 83493 +4, 83503 +5, 83491 +6, 83513 +7, 83517 +0, 83951 +1, 83935 +2, 83949 +3, 83955 +4, 83959 +5, 83949 +6, 83935 +7, 83949 +0, 84341 +1, 84325 +2, 84339 +3, 84345 +4, 84349 +5, 84339 +6, 84325 +7, 84339 +0, 85001 +1, 84985 +2, 84999 +3, 85005 +4, 85009 +5, 84999 +6, 84985 +7, 84999 +0, 95193 +1, 95123 +2, 95223 +3, 95055 +4, 95137 +5, 95285 +6, 95145 +7, 95277 +0, 113375 +1, 113151 +2, 114065 +3, 112895 +4, 112863 +5, 113273 +6, 113239 +7, 113381 +0, 116445 +1, 116331 +2, 116435 +3, 116221 +4, 116331 +5, 116435 +6, 116309 +7, 116435 +0, 116445 +1, 116331 +2, 116435 +3, 116221 +4, 116331 +5, 116435 +6, 116309 +7, 116435 +0, 115007 +1, 114893 +2, 114997 +3, 114783 +4, 114893 +5, 114997 +6, 114871 +7, 114997 +0, 193 +1, 113687 +2, 964 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.json new file mode 100644 index 0000000000000000000000000000000000000000..451ebad52224fd5b9cef99428c2c677c8cc55c0f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69826266112, + "active_bytes_current_delta": 48017935872, + "active_bytes_peak": 69826269184, + "active_bytes_peak_delta": 46331631616, + "after": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 69826269184, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 69826269184, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71596769280, + "reserved_bytes_peak": 71596769280, + "segment_count_current": 2596, + "segment_count_peak": 2596, + "timestamp_utc": "2026-08-30T00:05:00.071063Z", + "world_size": 8 + }, + "allocated_bytes_current": 69826266112, + "allocated_bytes_current_delta": 48017935872, + "allocated_bytes_peak": 69826269184, + "allocated_bytes_peak_delta": 46331631616, + "before": { + "active_bytes_current": 21808330240, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21808330240, + "allocated_bytes_peak": 23494637568, + "device": 0, + "inactive_split_bytes_current": 1876904448, + "inactive_split_bytes_peak": 1876904448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25293750272, + "reserved_bytes_peak": 25293750272, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:04:37.089170Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_current_delta": -129469952, + "inactive_split_bytes_peak": 13290654208, + "inactive_split_bytes_peak_delta": 11413749760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71596769280, + "reserved_bytes_current_delta": 46303019008, + "reserved_bytes_peak": 71596769280, + "reserved_bytes_peak_delta": 46303019008, + "segment_count_current": 2596, + "segment_count_current_delta": 1708, + "segment_count_peak": 2596, + "segment_count_peak_delta": 1708, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.319117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a035956b801c8836f401f1747c8e815657341ca7 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000500071517Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc903cec7faed9092e048a76c137ee22801fc96e5f24319e9a58c72d1a8e773b +size 12131150 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f8041c64f375a02bdf395395294276c41be1ef --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69826266112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91033299456, + "active_bytes_peak_delta": 21207030272, + "after": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91033299456, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91033299456, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:01.186332Z", + "world_size": 8 + }, + "allocated_bytes_current": 69826266112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91033299456, + "allocated_bytes_peak_delta": 21207030272, + "before": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 69826269184, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 69826269184, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71596769280, + "reserved_bytes_peak": 71596769280, + "segment_count_current": 2596, + "segment_count_peak": 2596, + "timestamp_utc": "2026-08-30T00:05:00.357820Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13290654208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_current_delta": 21770534912, + "reserved_bytes_peak": 93367304192, + "reserved_bytes_peak_delta": 21770534912, + "segment_count_current": 2766, + "segment_count_current_delta": 170, + "segment_count_peak": 2766, + "segment_count_peak_delta": 170, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.296821Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24d8a355645b4666d4954e87f81bdbf91e4e5852 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000501186736Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdd8bf7aea3b7679b0141a4e6fbf303d7f72437a93b318733d32e7bfa0fa4f6d +size 4883696 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8616b6b46d3d72b5e20ba58b9d09473b2b8f3d03 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69826266112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91042207232, + "active_bytes_peak_delta": 8907776, + "after": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91042207232, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91042207232, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:02.010118Z", + "world_size": 8 + }, + "allocated_bytes_current": 69826266112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91042207232, + "allocated_bytes_peak_delta": 8907776, + "before": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91033299456, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91033299456, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:01.306423Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13290654208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93367304192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2766, + "segment_count_current_delta": 0, + "segment_count_peak": 2766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.119749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9fa0d41263c0c72868094f2ecf42ca71208907fe --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000502010519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:402a45965d9202eabb1fc464ad54a5c8862c191657d991793dbbe46e247baebe +size 5419643 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8e80b51f4d6a638e11e814d5fc3df8aa487b93a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69826266112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91042207232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91042207232, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91042207232, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:02.850030Z", + "world_size": 8 + }, + "allocated_bytes_current": 69826266112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91042207232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91042207232, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91042207232, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:02.129227Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13290654208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93367304192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2766, + "segment_count_current_delta": 0, + "segment_count_peak": 2766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.958375Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0ccae731a7de51d705617deef61bf5443b1fc45 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000502850449Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb2fc4e0aed655d1ae86eb78c480e22b03a0d3c649b88e1ec237e1c119bd5bd +size 5419687 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17a12461ee872ce260ea682fe0fbf78fbecb8534 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69826266112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91042207232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91042207232, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91042207232, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:03.682497Z", + "world_size": 8 + }, + "allocated_bytes_current": 69826266112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91042207232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69826266112, + "active_bytes_peak": 91042207232, + "allocated_bytes_current": 69826266112, + "allocated_bytes_peak": 91042207232, + "device": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_peak": 13290654208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_peak": 93367304192, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-30T00:05:02.967940Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1747434496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13290654208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93367304192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93367304192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2766, + "segment_count_current_delta": 0, + "segment_count_peak": 2766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.789730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49aca04ade486472b2d9f4f797a25912e742b789 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000503682919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a414c55f792cf4c042659e9940f1a6cd42bc79b41d7d49b37ca127112445e6f1 +size 5419763 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4b770d36f2ee5a4154312971cdfd86352342c4c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69819414016, + "active_bytes_current_delta": 48007257088, + "active_bytes_peak": 69819417088, + "active_bytes_peak_delta": 46324779520, + "after": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 69819417088, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 69819417088, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:05:00.070909Z", + "world_size": 8 + }, + "allocated_bytes_current": 69819414016, + "allocated_bytes_current_delta": 48007257088, + "allocated_bytes_peak": 69819417088, + "allocated_bytes_peak_delta": 46324779520, + "before": { + "active_bytes_current": 21812156928, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21812156928, + "allocated_bytes_peak": 23494637568, + "device": 1, + "inactive_split_bytes_current": 1984226816, + "inactive_split_bytes_peak": 1984226816, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:04:37.088898Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_current_delta": -345283584, + "inactive_split_bytes_peak": 13480961024, + "inactive_split_bytes_peak_delta": 11496734208, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71479328768, + "reserved_bytes_current_delta": 46368030720, + "reserved_bytes_peak": 71479328768, + "reserved_bytes_peak_delta": 46368030720, + "segment_count_current": 2585, + "segment_count_current_delta": 1704, + "segment_count_peak": 2585, + "segment_count_peak_delta": 1704, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.326489Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d26221e927de17d5a6977ae984e32f6bf758dd2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260830T000500071406Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff722a55e6d5dab1811e4fe43908c4950899ded09a0773c3e8e2f1e738d26aa +size 10974361 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.json new file mode 100644 index 0000000000000000000000000000000000000000..408bacd35243ceb525688ed37687d560b8989745 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69819414016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91017127424, + "active_bytes_peak_delta": 21197710336, + "after": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91017127424, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91017127424, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:01.186288Z", + "world_size": 8 + }, + "allocated_bytes_current": 69819414016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91017127424, + "allocated_bytes_peak_delta": 21197710336, + "before": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 69819417088, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 69819417088, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2585, + "segment_count_peak": 2585, + "timestamp_utc": "2026-08-30T00:05:00.363261Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13480961024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 21768437760, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 21768437760, + "segment_count_current": 2754, + "segment_count_current_delta": 169, + "segment_count_peak": 2754, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.295585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..726b5e3ae651bb212e9db4af8b333f0dce72a7ef --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260830T000501186701Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a74784c408dc5195b50575802098856d865a9c9d537e25a285197f8527b90b +size 4805480 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61aca6372280bdea78b63f4c4876a716d9ab2e8d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69819414016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91038098944, + "active_bytes_peak_delta": 20971520, + "after": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91038098944, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91038098944, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:02.010059Z", + "world_size": 8 + }, + "allocated_bytes_current": 69819414016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91038098944, + "allocated_bytes_peak_delta": 20971520, + "before": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91017127424, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91017127424, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:01.304493Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13480961024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.116051Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29ca77ca80d4a6e482c22097012f96e77404acc0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260830T000502010461Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1a92a1cbe6b8a3eaa248f89b943a7eae926a1141d2a59106f2b2210d5639f8 +size 4815924 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e770ad7b873da3496a34911aad59f73d49aeac89 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69819414016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91038098944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91038098944, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91038098944, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:02.849954Z", + "world_size": 8 + }, + "allocated_bytes_current": 69819414016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91038098944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91038098944, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91038098944, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:02.124369Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13480961024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.956976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1b37a6267b40c3cb8cd34e9824c23983cd6094d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260830T000502850354Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7edba1fc6477d0231187205c4e06dac3b0b92b21700711b9d7e1fdb6db5b10c +size 4816089 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5eb5f5f300d8aa2d78c157764d782399576825f3 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69819414016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91038098944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91038098944, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91038098944, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:03.682460Z", + "world_size": 8 + }, + "allocated_bytes_current": 69819414016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91038098944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69819414016, + "active_bytes_peak": 91038098944, + "allocated_bytes_current": 69819414016, + "allocated_bytes_peak": 91038098944, + "device": 1, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_peak": 13480961024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-30T00:05:02.965156Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1638943232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13480961024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.786188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25d3af49c1e5d844e8191522aab3d208c5692ee6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260830T000503682875Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c0fb0f5449084c583582d517aab454fba6663e548f1b5fc394f275eb7b024e +size 4815991 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15988622ca6aec6cd09a71b5c7965d39a692dd75 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 48011968000, + "active_bytes_peak": 69822242304, + "active_bytes_peak_delta": 46327604736, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.070874Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 48011968000, + "allocated_bytes_peak": 69822242304, + "allocated_bytes_peak_delta": 46327604736, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 2, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:04:37.089357Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": -102530560, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 11469995008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71607255040, + "reserved_bytes_current_delta": 46284144640, + "reserved_bytes_peak": 71607255040, + "reserved_bytes_peak_delta": 46284144640, + "segment_count_current": 2591, + "segment_count_current_delta": 1703, + "segment_count_peak": 2591, + "segment_count_peak_delta": 1703, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.316104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fadda0f2a3bbb655bfb8c7151a5622e3dd38795 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260830T000500071404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddd440d5080cd14d516bee0cdd200d2d0893c06ef67e8c61d640978ff16aabaf +size 12119262 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae864570da84a2fe98d5bb88886a2c355ca73beb --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91021637120, + "active_bytes_peak_delta": 21199394816, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.186247Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91021637120, + "allocated_bytes_peak_delta": 21199394816, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.354706Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 21749563392, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 21749563392, + "segment_count_current": 2751, + "segment_count_current_delta": 160, + "segment_count_peak": 2751, + "segment_count_peak_delta": 160, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.295452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14179d74a60e13b192f4d66937c7d1099806848e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260830T000501186657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13841e186773d7b669e0a8b637cb728ddc8687c01d8b71d854dd17758fb6db19 +size 4874645 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b7ce2169eec6036fae94578f9e8bea7585dbe92f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 17722880, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.010013Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 17722880, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.305095Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.117932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32e1deee41aeee568411fa01123907b0311b9faf --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260830T000502010418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7359cc727998ad95c4c86e1d99d0a26989ef61a2a37d815d5a3cafe3ef754a4 +size 5410916 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2c8c1c04e67e503ec1694cc228b02138025a1eb --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.849924Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.127343Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.956598Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c047186428150adb2c52adfc540724ec25e735d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260830T000502850329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112cd009b8913b4ae82c30d7f7314a8c7cbb1aa431bdf6d5f8bcc12264c26d29 +size 5410951 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..909228905057c511646dd89e88356b2073f856e1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:03.682415Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 2, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.965799Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.788521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7c43fc252efc07023646c2eb325ec3aeff01c7f --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260830T000503682837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761750444e9bc9e2ba24c27b1535e7f692846271ac359f6fe889af39ca448fd6 +size 5410952 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2f45a517418d684a887f416e037a7ec2d6d8341 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69818953728, + "active_bytes_current_delta": 48002565120, + "active_bytes_peak": 69818956800, + "active_bytes_peak_delta": 46324319232, + "after": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 69818956800, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 69818956800, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71361888256, + "reserved_bytes_peak": 71361888256, + "segment_count_current": 2575, + "segment_count_peak": 2575, + "timestamp_utc": "2026-08-30T00:05:00.070866Z", + "world_size": 8 + }, + "allocated_bytes_current": 69818953728, + "allocated_bytes_current_delta": 48002565120, + "allocated_bytes_peak": 69818956800, + "allocated_bytes_peak_delta": 46324319232, + "before": { + "active_bytes_current": 21816388608, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21816388608, + "allocated_bytes_peak": 23494637568, + "device": 3, + "inactive_split_bytes_current": 1919177728, + "inactive_split_bytes_peak": 1919177728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:04:37.089124Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_current_delta": -399311872, + "inactive_split_bytes_peak": 13229369856, + "inactive_split_bytes_peak_delta": 11310192128, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71361888256, + "reserved_bytes_current_delta": 46250590208, + "reserved_bytes_peak": 71361888256, + "reserved_bytes_peak_delta": 46250590208, + "segment_count_current": 2575, + "segment_count_current_delta": 1694, + "segment_count_peak": 2575, + "segment_count_peak_delta": 1694, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.333032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03dbe96e54c602850c8791e4c1ae0d200a975c6c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260830T000500071407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcc5f09ffc888db87d79c38a99de5acfb4256093fe577524f01fcbdfaacade0 +size 14457499 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d677c5db3e75f35effb9cfea0dcf1b7acc25be9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69818953728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91032886272, + "active_bytes_peak_delta": 21213929472, + "after": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91032886272, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91032886272, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:01.186256Z", + "world_size": 8 + }, + "allocated_bytes_current": 69818953728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91032886272, + "allocated_bytes_peak_delta": 21213929472, + "before": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 69818956800, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 69818956800, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71361888256, + "reserved_bytes_peak": 71361888256, + "segment_count_current": 2575, + "segment_count_peak": 2575, + "timestamp_utc": "2026-08-30T00:05:00.372843Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13229369856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93132423168, + "reserved_bytes_current_delta": 21770534912, + "reserved_bytes_peak": 93132423168, + "reserved_bytes_peak_delta": 21770534912, + "segment_count_current": 2745, + "segment_count_current_delta": 170, + "segment_count_peak": 2745, + "segment_count_peak_delta": 170, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.295241Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d67154a140da5cfbe52be3233376f0e7db4f8e82 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260830T000501186672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bed627d874a09b71bd97078d7531b7ac9ebf0b4035d75e169d7539a8a3b91448 +size 5005665 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a99b0648854ba4908ff73138a36164181e47d31 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69818953728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91033321984, + "active_bytes_peak_delta": 435712, + "after": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91033321984, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91033321984, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:02.010033Z", + "world_size": 8 + }, + "allocated_bytes_current": 69818953728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91033321984, + "allocated_bytes_peak_delta": 435712, + "before": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91032886272, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91032886272, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:01.304919Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13229369856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93132423168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93132423168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2745, + "segment_count_current_delta": 0, + "segment_count_peak": 2745, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.118784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..235de4d03cccc2bc98d0169aa854c0a094aa74f1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260830T000502010433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e2c8562fe1aab49d43be545867dc9e1d9f3aad55e9df9b197ee50935a395981 +size 5541743 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a67c015d8aef8faed5480100b38776abe4778b1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69818953728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91033321984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91033321984, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91033321984, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:02.849907Z", + "world_size": 8 + }, + "allocated_bytes_current": 69818953728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91033321984, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91033321984, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91033321984, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:02.128113Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13229369856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93132423168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93132423168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2745, + "segment_count_current_delta": 0, + "segment_count_peak": 2745, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.959144Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ad03ed2678a37442d758336877ffac56ab0cdea --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260830T000502850313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df5afdea24f7def06d3c30ae59dfa6bdfc6c4191bc786d395e0548b53d53ed79 +size 5541708 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e4dc18bd36be39a25139bf70b8f005906bf062c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69818953728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91033321984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91033321984, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91033321984, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:03.682412Z", + "world_size": 8 + }, + "allocated_bytes_current": 69818953728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91033321984, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69818953728, + "active_bytes_peak": 91033321984, + "allocated_bytes_current": 69818953728, + "allocated_bytes_peak": 91033321984, + "device": 3, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_peak": 13229369856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93132423168, + "reserved_bytes_peak": 93132423168, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-30T00:05:02.968457Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1519865856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13229369856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93132423168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93132423168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2745, + "segment_count_current_delta": 0, + "segment_count_peak": 2745, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.788485Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c0bf8b51eb19ddf904685e512bc7a969e6b25b5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260830T000503682825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67cf6c33f46930f3e19de1b13501d1ecd79a5634abd15b264d26ae7dea84eaa4 +size 5541737 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43798a1cb7025b63ad98af396261de9aaaf72955 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69830206976, + "active_bytes_current_delta": 48022231040, + "active_bytes_peak": 69830210048, + "active_bytes_peak_delta": 46335572480, + "after": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 69830210048, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 69830210048, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2588, + "segment_count_peak": 2588, + "timestamp_utc": "2026-08-30T00:05:00.070917Z", + "world_size": 8 + }, + "allocated_bytes_current": 69830206976, + "allocated_bytes_current_delta": 48022231040, + "allocated_bytes_peak": 69830210048, + "allocated_bytes_peak_delta": 46335572480, + "before": { + "active_bytes_current": 21807975936, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21807975936, + "allocated_bytes_peak": 23494637568, + "device": 4, + "inactive_split_bytes_current": 1965339136, + "inactive_split_bytes_peak": 1965339136, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:04:37.088997Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_current_delta": -337188864, + "inactive_split_bytes_peak": 13429277184, + "inactive_split_bytes_peak_delta": 11463938048, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71479328768, + "reserved_bytes_current_delta": 46368030720, + "reserved_bytes_peak": 71479328768, + "reserved_bytes_peak_delta": 46368030720, + "segment_count_current": 2588, + "segment_count_current_delta": 1707, + "segment_count_peak": 2588, + "segment_count_peak_delta": 1707, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.331687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f516c1769a2ad7d2759eb804805f322e40da09ff --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260830T000500071407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c375a67f01e1e7b5284d36e4b907b5f723de2e160b1af1f30964df60b2bfd3b +size 14460366 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a5c93c4de35ef74eb741fb4d9ce1278e2f04012 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69830206976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91026178048, + "active_bytes_peak_delta": 21195968000, + "after": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91026178048, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91026178048, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:01.186271Z", + "world_size": 8 + }, + "allocated_bytes_current": 69830206976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91026178048, + "allocated_bytes_peak_delta": 21195968000, + "before": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 69830210048, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 69830210048, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71479328768, + "reserved_bytes_peak": 71479328768, + "segment_count_current": 2588, + "segment_count_peak": 2588, + "timestamp_utc": "2026-08-30T00:05:00.370305Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13429277184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 21768437760, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 21768437760, + "segment_count_current": 2757, + "segment_count_current_delta": 169, + "segment_count_peak": 2757, + "segment_count_peak_delta": 169, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.296761Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ba03fb0e5d20425b4ca6c51e78666e691096714 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260830T000501186685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f90d704a55a06e7f6dbfa396f4e28b316c609cf5ce77f34b7a21f4a2180cd2 +size 4885921 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d43db50c77b8ebec840170d6bc870bd6dedec241 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69830206976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91047149568, + "active_bytes_peak_delta": 20971520, + "after": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91047149568, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91047149568, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:02.010022Z", + "world_size": 8 + }, + "allocated_bytes_current": 69830206976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91047149568, + "allocated_bytes_peak_delta": 20971520, + "before": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91026178048, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91026178048, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:01.306481Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13429277184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2757, + "segment_count_current_delta": 0, + "segment_count_peak": 2757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.119491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6352d15a43fb234d2f07a712d15219bf89ddfcc --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000502010429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0376ebfecba59dade39d74cdcb3926f4333728ddfaf8b61261b46308b3f4f6c0 +size 5421640 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.json new file mode 100644 index 0000000000000000000000000000000000000000..057d663fd6a09f26d7545af962503219a9254cd5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69830206976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91047149568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91047149568, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91047149568, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:02.849890Z", + "world_size": 8 + }, + "allocated_bytes_current": 69830206976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91047149568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91047149568, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91047149568, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:02.127973Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13429277184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2757, + "segment_count_current_delta": 0, + "segment_count_peak": 2757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.962396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f839ec5b5bf39169b22d5de0645a4d8e12c84ed --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000502850298Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97a90501a2a8c6c377da7f2ef7cb97919fba4c9900df2460976c466f0f7dc4b +size 5421805 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba988deb18c33b586cf90119f79a30c6b613418 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69830206976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91047149568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91047149568, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91047149568, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:03.682410Z", + "world_size": 8 + }, + "allocated_bytes_current": 69830206976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91047149568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69830206976, + "active_bytes_peak": 91047149568, + "allocated_bytes_current": 69830206976, + "allocated_bytes_peak": 91047149568, + "device": 4, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_peak": 13429277184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93247766528, + "reserved_bytes_peak": 93247766528, + "segment_count_current": 2757, + "segment_count_peak": 2757, + "timestamp_utc": "2026-08-30T00:05:02.970789Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1628150272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13429277184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93247766528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93247766528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2757, + "segment_count_current_delta": 0, + "segment_count_peak": 2757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.788985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bc72d6e65259c785e7869fc82e292d5092f595b --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000503682833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e74a995c7fb552c89e57e01d8c9b45ea057166525118acee050d261e520e391 +size 5421707 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.json new file mode 100644 index 0000000000000000000000000000000000000000..261b76c31a9a3a7ce87de8c756411af16aa466c1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 48011968000, + "active_bytes_peak": 69822242304, + "active_bytes_peak_delta": 46327604736, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.070855Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 48011968000, + "allocated_bytes_peak": 69822242304, + "allocated_bytes_peak_delta": 46327604736, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 5, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:04:37.089100Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": -102530560, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 11469995008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71607255040, + "reserved_bytes_current_delta": 46284144640, + "reserved_bytes_peak": 71607255040, + "reserved_bytes_peak_delta": 46284144640, + "segment_count_current": 2591, + "segment_count_current_delta": 1703, + "segment_count_peak": 2591, + "segment_count_peak_delta": 1703, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.328095Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..691f185c312d133e3b01be6b64208b1d4e6f9541 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000500071393Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf0da4b4567013761d64022c09c31e3aead6d42d0a5b228f1d436b6873e8dfc +size 12119259 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ee3bb97a022f75f69dc17455851e5a9611af7cd --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91021637120, + "active_bytes_peak_delta": 21199394816, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.186226Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91021637120, + "allocated_bytes_peak_delta": 21199394816, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.365065Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 21749563392, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 21749563392, + "segment_count_current": 2751, + "segment_count_current_delta": 160, + "segment_count_peak": 2751, + "segment_count_peak_delta": 160, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.295036Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cba06703b1236c368cf6705ef4bef75bfa687fa6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000501186654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f2ecdf0eed98eda48bb0a5e7e8553414d3b559b4071a7ba9afea0e1f09a4390 +size 4874641 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4df8b39b09f75adf72b71762b0aac5956eaf0535 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 17722880, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.009998Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 17722880, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.303965Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.115751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96c3aa26e73d835f0498eaec6486dbe3201d01f1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000502010411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c947c4119190f63e163dee941139a0d5d38afcd4deb25eee9b4a70383f03a5b1 +size 5410912 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b37bdd9f5264c0c10a166ac3a7461d936b62447 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.849887Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.124063Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.954556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2307c22fa888cecc3d1961c62c14c1f6ff56a4dd --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000502850296Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a043a0e67cf96ec097f84fad98167463f74d9431dcae7428b5dd6c61342a91c +size 5410947 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.json new file mode 100644 index 0000000000000000000000000000000000000000..976fc874fd1d8c140f53b6270e1f296cf85e4e80 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:03.682431Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 5, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.962862Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.787258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..716b384e1129bf67dd8463ae6ce54e6a8113e341 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000503682843Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca8091379c7a5762db94210d5b9f4cbc241cd5d404d242ab58f33ccc45e9ddc +size 5410948 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3265f61e8a49790f71bb76f70b102502bc71b650 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69809795584, + "active_bytes_current_delta": 47997638656, + "active_bytes_peak": 69809798656, + "active_bytes_peak_delta": 46315161088, + "after": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 69809798656, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 69809798656, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71454162944, + "reserved_bytes_peak": 71454162944, + "segment_count_current": 2583, + "segment_count_peak": 2583, + "timestamp_utc": "2026-08-30T00:05:00.070948Z", + "world_size": 8 + }, + "allocated_bytes_current": 69809795584, + "allocated_bytes_current_delta": 47997638656, + "allocated_bytes_peak": 69809798656, + "allocated_bytes_peak_delta": 46315161088, + "before": { + "active_bytes_current": 21812156928, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21812156928, + "allocated_bytes_peak": 23494637568, + "device": 6, + "inactive_split_bytes_current": 1986323968, + "inactive_split_bytes_peak": 1986323968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25111298048, + "reserved_bytes_peak": 25111298048, + "segment_count_current": 881, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-30T00:04:37.089034Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_current_delta": -365025280, + "inactive_split_bytes_peak": 13399172096, + "inactive_split_bytes_peak_delta": 11412848128, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71454162944, + "reserved_bytes_current_delta": 46342864896, + "reserved_bytes_peak": 71454162944, + "reserved_bytes_peak_delta": 46342864896, + "segment_count_current": 2583, + "segment_count_current_delta": 1702, + "segment_count_peak": 2583, + "segment_count_peak_delta": 1702, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.314337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e00feb43eeec2c1944779126e500de62122657d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000500071472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c42e0ffd40370d2fb378a29a3c4dbb0746e42949074454e7daa997b41d1f5889 +size 10975153 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2e5cd4b342d3e7358873961bdd8c4d9c815fd77 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69809795584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91007535104, + "active_bytes_peak_delta": 21197736448, + "after": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91007535104, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91007535104, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:01.186272Z", + "world_size": 8 + }, + "allocated_bytes_current": 69809795584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91007535104, + "allocated_bytes_peak_delta": 21197736448, + "before": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 69809798656, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 69809798656, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71454162944, + "reserved_bytes_peak": 71454162944, + "segment_count_current": 2583, + "segment_count_peak": 2583, + "timestamp_utc": "2026-08-30T00:05:00.350820Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13399172096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93224697856, + "reserved_bytes_current_delta": 21770534912, + "reserved_bytes_peak": 93224697856, + "reserved_bytes_peak_delta": 21770534912, + "segment_count_current": 2753, + "segment_count_current_delta": 170, + "segment_count_peak": 2753, + "segment_count_peak_delta": 170, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.295811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6addd780427f979b6216259f3886416df35c9b7d --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000501186689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:598cc34c0c0297c4ae9f66fd014c630b329e19939e2521c68d6cba90316ecfca +size 4806528 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498b609d1187414a42b1a95d9ae453ef2a80b8a7 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69809795584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91028506624, + "active_bytes_peak_delta": 20971520, + "after": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91028506624, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91028506624, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:02.010011Z", + "world_size": 8 + }, + "allocated_bytes_current": 69809795584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91028506624, + "allocated_bytes_peak_delta": 20971520, + "before": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91007535104, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91007535104, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:01.305444Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13399172096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93224697856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93224697856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2753, + "segment_count_current_delta": 0, + "segment_count_peak": 2753, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.119348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..799275b2f944dcc0e4b65639a77fa6bbba1727d0 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000502010417Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b511121e75a5e7f936e7b16b1fdb5f7cb93f82193557f930f01f61df2ad0299e +size 4817201 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a19a576c1c574ebb03affbad3e437eb0cd6020a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69809795584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91028506624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91028506624, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91028506624, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:02.849901Z", + "world_size": 8 + }, + "allocated_bytes_current": 69809795584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91028506624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91028506624, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91028506624, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:02.127816Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13399172096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93224697856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93224697856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2753, + "segment_count_current_delta": 0, + "segment_count_peak": 2753, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.957275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..adb3de8f808942dd3eedaf5676af04a5ef964a79 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000502850298Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e423249361a274718ea58d849bee6de5624267c8bb2097c0eaba23a12fa89fa +size 4817166 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.json new file mode 100644 index 0000000000000000000000000000000000000000..462e03e3804254fea1ea41fd6a71851e8b516f83 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69809795584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91028506624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91028506624, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91028506624, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:03.682401Z", + "world_size": 8 + }, + "allocated_bytes_current": 69809795584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91028506624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69809795584, + "active_bytes_peak": 91028506624, + "allocated_bytes_current": 69809795584, + "allocated_bytes_peak": 91028506624, + "device": 6, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_peak": 13399172096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93224697856, + "reserved_bytes_peak": 93224697856, + "segment_count_current": 2753, + "segment_count_peak": 2753, + "timestamp_utc": "2026-08-30T00:05:02.965438Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1621298688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13399172096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93224697856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93224697856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2753, + "segment_count_current_delta": 0, + "segment_count_peak": 2753, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.788287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bd2e32cd6f57d0c29d9c6215a46435979337339 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000503682817Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ea271a16f0792ca82928c0e3c808bdbc3693d8d5561e67d311867eb5adc9828 +size 4817195 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..727c61e72c064e972be669a3144a8080fcd5cda9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 48011968000, + "active_bytes_peak": 69822242304, + "active_bytes_peak_delta": 46327604736, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.070926Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 48011968000, + "allocated_bytes_peak": 69822242304, + "allocated_bytes_peak_delta": 46327604736, + "before": { + "active_bytes_current": 21810271232, + "active_bytes_peak": 23494637568, + "allocated_bytes_current": 21810271232, + "allocated_bytes_peak": 23494637568, + "device": 7, + "inactive_split_bytes_current": 1864477696, + "inactive_split_bytes_peak": 1864477696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25323110400, + "reserved_bytes_peak": 25323110400, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-30T00:04:37.088929Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": -102530560, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 11469995008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71607255040, + "reserved_bytes_current_delta": 46284144640, + "reserved_bytes_peak": 71607255040, + "reserved_bytes_peak_delta": 46284144640, + "segment_count_current": 2591, + "segment_count_current_delta": 1703, + "segment_count_peak": 2591, + "segment_count_peak_delta": 1703, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.pickle", + "timestamp_utc": "2026-08-30T00:05:00.313845Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac26a5bb954bd17db8cb1b31d765c148b8d801df --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000500071403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1b8a45612c861c26af1d047a0b32b77a2b81d7dae878e3a208326fc15e2e89 +size 12119262 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c45225a7e5a0c86393602f71788e094c7c93d384 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91021637120, + "active_bytes_peak_delta": 21199394816, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.186319Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91021637120, + "allocated_bytes_peak_delta": 21199394816, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 69822242304, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 69822242304, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71607255040, + "reserved_bytes_peak": 71607255040, + "segment_count_current": 2591, + "segment_count_peak": 2591, + "timestamp_utc": "2026-08-30T00:05:00.357021Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 21749563392, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 21749563392, + "segment_count_current": 2751, + "segment_count_current_delta": 160, + "segment_count_peak": 2751, + "segment_count_peak_delta": 160, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.pickle", + "timestamp_utc": "2026-08-30T00:05:01.291541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0537342310ab2813cf2ffd894ec1c736a5fe6376 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000501186733Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a26910b2f61e2d3dc8249f2aee7d2088d2b7811cf4db8daeceab5b54cde5050e +size 4874645 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8954dde47f1421187009e3d569b0a02e2f942c1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 17722880, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.010072Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 17722880, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91021637120, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91021637120, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:01.299223Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.105484Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4e30dc079f7746b540aaecfdeeb601279fa3e4e --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000502010468Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e46eb58beb8abcc17ac9371a6a0a1ca91243aaf341a9adaf3b8e391437308c07 +size 5410916 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.json new file mode 100644 index 0000000000000000000000000000000000000000..018dbaef3c0eca84233a1222523451b5f31e232c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.849962Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.112900Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.pickle", + "timestamp_utc": "2026-08-30T00:05:02.943827Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e617435b7885d9bc8694b12873003618e10f00a6 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000502850350Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6ad4b928de6273d201de6274838c8c75b44cf2dbf8ba5e38a51455ab2e64eb +size 5410951 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.json b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aac4596f730a854cd9986db90d13b22360c612c --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69822239232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 91039360000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:03.682434Z", + "world_size": 8 + }, + "allocated_bytes_current": 69822239232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 91039360000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69822239232, + "active_bytes_peak": 91039360000, + "allocated_bytes_current": 69822239232, + "allocated_bytes_peak": 91039360000, + "device": 7, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_peak": 13334472704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93356818432, + "reserved_bytes_peak": 93356818432, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-30T00:05:02.951180Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1761947136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13334472704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93356818432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93356818432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.pickle", + "timestamp_utc": "2026-08-30T00:05:03.776378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.pickle b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6959b468a560662f592fa94b41e9d1757bc15418 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000503682837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cecfc20b27e199c462752d6d7ed03c84c2e4aff35846441927f5331e4b0a1e7 +size 5410952 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8e6368e152ba11e697530585fe4ec38a783929f1 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_better_memory_dump_support +commit=51c387c7 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=0 +async_grad_reduce=1 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b0f744f4022e48e35c90ae28376c6127973f3839 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log @@ -0,0 +1,2731 @@ +[00:04:06] WARN: moonep_backend_async_grad_reduce_better_memory_dump_support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 51c387c7 +============================================================== +W0830 00:04:07.620000 276110 torch/distributed/run.py:766] +W0830 00:04:07.620000 276110 torch/distributed/run.py:766] ***************************************** +W0830 00:04:07.620000 276110 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 00:04:07.620000 276110 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 0 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to [Gloo] Rank 7 peer ranks. 1Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +7[Gloo] Rank is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W830 00:04:25.657881837 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 00:04:25.661572264 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 00:04:25.661701760 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 00:04:25.661708462 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 00:04:25.661809196 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 00:04:25.661839573 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 00:04:25.663405427 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 00:04:25.818208993 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.704 seconds +time to initialize megatron (seconds): 7.022 +[after megatron is initialized] datetime: 2026-08-30 00:04:31 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.linear_proj.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.mlp.router.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.q_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 00:04:36 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 00:04:37 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5385.58, 5388.76) + train/valid/test-data-iterators-setup ..........: (126.76, 138.72) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 00:04:37 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 144.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 147.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [0, 2048, 1408, 1536, 3968, 2944, 4224, 896, 2816, 1152, 2432, 2176, 896, 2560, 1152, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [110, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [71, 77, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-30 00:05:00] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 22983.8 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.353 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66591.5166015625 | max allocated: 66591.51708984375 | reserved: 68280.0 | max reserved: 68280.0 + [2026-08-30 00:05:01] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 1115.8 | throughput per GPU (TFLOP/s/GPU): 79.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.502 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:05:02] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 823.7 | throughput per GPU (TFLOP/s/GPU): 108.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008712E+01 | loss scale: 1.0 | grad norm: 180.582 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:05:02] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 839.8 | throughput per GPU (TFLOP/s/GPU): 106.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604689E+00 | loss scale: 1.0 | grad norm: 8.728 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 00:05:03] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 832.4 | throughput per GPU (TFLOP/s/GPU): 107.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307743E+00 | loss scale: 1.0 | grad norm: 5.187 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 00:05:03 +[rank0]:[W830 00:05:05.273686375 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 00:05:06.015976279 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 00:05:06.020981765 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 00:05:06.043601576 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 00:05:06.195282797 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 00:05:06.210946059 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 00:05:07.555403489 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 00:05:07.557442710 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 00:05:07.577000 276110 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 00:05:07.577000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276299 closing signal SIGTERM +W0830 00:05:07.578000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276300 closing signal SIGTERM +W0830 00:05:07.579000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276301 closing signal SIGTERM +W0830 00:05:07.580000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276302 closing signal SIGTERM +W0830 00:05:07.581000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276303 closing signal SIGTERM +W0830 00:05:07.582000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276304 closing signal SIGTERM +W0830 00:05:07.582000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276305 closing signal SIGTERM +W0830 00:05:07.583000 276110 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276306 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 276110 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/results.tsv b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d58882ef6eb5c8d7ed37745094c7e7a030e99655 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/results.tsv @@ -0,0 +1,6 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +QWEN3_235B moonep matched 16 8192 8 none none 1 OK 341.70 341.70 120303 /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async__memiteration__it6/rep1 +QWEN3_235B moonep matched 16 8192 8 none none 1 OK NA NA 120131 /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 1 OK 435.30 435.30 116703 /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 1 OK NA NA 116445 /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__nocg__async__memiteration__it5/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 622.80 622.80 133311 /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async__memiteration__it6/rep1