diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ced37a4d6e27d3222b3f5623d9d9a292442ee117 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901254656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113456552448, + "active_bytes_peak_delta": 30836036096, + "after": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 113456552448, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 113456552448, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113873256448, + "reserved_bytes_peak": 113873256448, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:37.348676Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901254656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113456552448, + "allocated_bytes_peak_delta": 30836036096, + "before": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 82620516352, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 82620516352, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 83022053376, + "reserved_bytes_peak": 83022053376, + "segment_count_current": 2125, + "segment_count_peak": 2125, + "timestamp_utc": "2026-08-26T19:17:36.067939Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14139202560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113873256448, + "reserved_bytes_current_delta": 30851203072, + "reserved_bytes_peak": 113873256448, + "reserved_bytes_peak_delta": 30851203072, + "segment_count_current": 2836, + "segment_count_current_delta": 711, + "segment_count_peak": 2836, + "segment_count_peak_delta": 711, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.476498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84ed548de13496535639ed149a876bd4e722ea34 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd22d0b31afd0255b04015d9a7b4e9df333063e40f147439b23bec488ad4cdcf +size 7113911 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c21a8c0ddea8d73f92da1a2a9938585d8de78ff --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901254656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113495366144, + "active_bytes_peak_delta": 38813696, + "after": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113873256448, + "reserved_bytes_peak": 113873256448, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:38.517239Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901254656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113495366144, + "allocated_bytes_peak_delta": 38813696, + "before": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 113456552448, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 113456552448, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113873256448, + "reserved_bytes_peak": 113873256448, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:37.492292Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14139202560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113873256448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113873256448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2836, + "segment_count_current_delta": 0, + "segment_count_peak": 2836, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.647254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bad3ecc674012fe1035a00f66ea4c48c19a9089c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:982257ddd1e947c99f2570403f65f61ca2f011471cd1462fb9c5af7cffa8c375 +size 6892313 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7caf9c667d896c2b799ba10cc5fc579c5d2d1afc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918298112, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113495366144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918298112, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75918298112, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_peak": 15467503104, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119042736128, + "reserved_bytes_peak": 119042736128, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:17:41.821331Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918298112, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113495366144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113873256448, + "reserved_bytes_peak": 113873256448, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:38.662900Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15467503104, + "inactive_split_bytes_peak_delta": 1328300544, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119042736128, + "reserved_bytes_current_delta": 5169479680, + "reserved_bytes_peak": 119042736128, + "reserved_bytes_peak_delta": 5169479680, + "segment_count_current": 3126, + "segment_count_current_delta": 290, + "segment_count_peak": 3126, + "segment_count_peak_delta": 290, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.960602Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2104ca3d1c772bf1800e29920affd0b99a4f2c7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7488c72f8849c7acc6825f47d531d1216b71cfd482bf98b9a9258ad406183644 +size 7508905 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e6ec05a21f56fdddfee930b8224db235b89a778 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918298112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113495366144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918298112, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75918298112, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_peak": 15467503104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119042736128, + "reserved_bytes_peak": 119042736128, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:17:42.351495Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918298112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113495366144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75918298112, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75918298112, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_peak": 15467503104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119042736128, + "reserved_bytes_peak": 119042736128, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:17:41.977258Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15467503104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119042736128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119042736128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3126, + "segment_count_current_delta": 0, + "segment_count_peak": 3126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.371970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b05af9bd430655981d6fdbe5171e1d16239714f5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92651b2e399d31cbb87b64130deeff9d01389b8e0872afd4185c48f5b885c6f6 +size 902755 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed20fa01f2b0909e31b92431019eb525d2ab22f9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918298112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113495366144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918298112, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75918298112, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_peak": 15467503104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119042736128, + "reserved_bytes_peak": 119042736128, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:17:42.732476Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918298112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113495366144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75918298112, + "active_bytes_peak": 113495366144, + "allocated_bytes_current": 75918298112, + "allocated_bytes_peak": 113495366144, + "device": 1, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_peak": 15467503104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119042736128, + "reserved_bytes_peak": 119042736128, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:17:42.373557Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5950321664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15467503104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119042736128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119042736128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3126, + "segment_count_current_delta": 0, + "segment_count_peak": 3126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751054Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcc86ddaca8f2638063c8754c2ef17c5293a4b56 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87358302e71f8050e4d1f4e0c8b125bcd8f2e0539bc811ebd33be2336b197cb5 +size 902755 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71faff767bc45881c2812d769e7dfd5e272f29b2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 32328931328, + "active_bytes_peak": 82624266240, + "active_bytes_peak_delta": 39063780352, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:35.910227Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 32328931328, + "allocated_bytes_peak": 82624266240, + "allocated_bytes_peak_delta": 39063780352, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 2, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:17:09.289391Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 4645955584, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 12537032192, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 82996887552, + "reserved_bytes_current_delta": 38161874944, + "reserved_bytes_peak": 82996887552, + "reserved_bytes_peak_delta": 38161874944, + "segment_count_current": 2112, + "segment_count_current_delta": 1229, + "segment_count_peak": 2112, + "segment_count_peak_delta": 1229, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.047916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e41e934fb56e5e9e113673080d4c33df966ea3bc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feb161bcba0cd15af96cbddd476e5bbbbcc7702ae29e002495a161d94622b2b6 +size 7001386 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e63b1c531ed2821a6f8ea2a233be586335a1884d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113455617536, + "active_bytes_peak_delta": 30831351296, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.348673Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113455617536, + "allocated_bytes_peak_delta": 30831351296, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:36.069015Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 30844911616, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 30844911616, + "segment_count_current": 2812, + "segment_count_current_delta": 700, + "segment_count_peak": 2812, + "segment_count_peak_delta": 700, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.477367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68f109c10be47015f1a51af9d29a4901a62b69a6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb407cb73c7f6b58ae60b7500ac00821299baa5e1cabe4319498795df9c8c375 +size 6492477 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbcf7b12eb3a88deb640a12007ae724bdbd4b8d7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 43040768, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.517252Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 43040768, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.492866Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2812, + "segment_count_current_delta": 0, + "segment_count_peak": 2812, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.646112Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2506a3afb6fca95f07cedd5d40b4d00029ff871f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b8e83304e96a7c1cdebf66dfd35177382faa8f65936acf9ca3f1ab51cd3c30 +size 6457970 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b0ee2747cdae1416f9310a49e6c3bd5fad7e006 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.821366Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.661216Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 1306978304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 5171576832, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 5171576832, + "segment_count_current": 3111, + "segment_count_current_delta": 299, + "segment_count_peak": 3111, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.961683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90ceead675ca765426c5f80fbd5a558faa212a44 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ba6b6f670b2f05f5b3a2ba8a6b103fc236a236a656023fe21288c42e3b6458 +size 7491891 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82fd631b28cf451786180e968861b4b5e9d1c7f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.351498Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.978110Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..153c4cd70ed9a9b84d805993288110c5b8956383 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4e61955093733498e6a3566b7a8c688e2dc8fe535215b82c6c6ecefde0af079 +size 886237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..576c8e5562049ca58fcd49193abe0a3564db37b4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.732501Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 2, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.373743Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751076Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..855001daa3a78e601c5d7cf4d9844ddadc0290cd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05dc5dc4571eeeccdfe5f49c6101cb5aa84f61118e3598b3855f030e6e837266 +size 886237 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63b4d89516388cadfe825ac7bbecc9cf0c89fe36 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906373632, + "active_bytes_current_delta": 32348899840, + "active_bytes_peak": 82622775808, + "active_bytes_peak_delta": 39059010560, + "after": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 82622775808, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 82622775808, + "device": 3, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_peak": 14129433600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 83022053376, + "reserved_bytes_peak": 83022053376, + "segment_count_current": 2127, + "segment_count_peak": 2127, + "timestamp_utc": "2026-08-26T19:17:35.910251Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906373632, + "allocated_bytes_current_delta": 32348899840, + "allocated_bytes_peak": 82622775808, + "allocated_bytes_peak_delta": 39059010560, + "before": { + "active_bytes_current": 43557473792, + "active_bytes_peak": 43563765248, + "allocated_bytes_current": 43557473792, + "allocated_bytes_peak": 43563765248, + "device": 3, + "inactive_split_bytes_current": 1223012864, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T19:17:09.289271Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_current_delta": 4701484544, + "inactive_split_bytes_peak": 14129433600, + "inactive_split_bytes_peak_delta": 12517172224, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 83022053376, + "reserved_bytes_current_delta": 38226886656, + "reserved_bytes_peak": 83022053376, + "reserved_bytes_peak_delta": 38226886656, + "segment_count_current": 2127, + "segment_count_current_delta": 1240, + "segment_count_peak": 2127, + "segment_count_peak_delta": 1240, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.045282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42c5363c985e6c70aaf56d156d0316310ccefbc5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b3a54f0eb6432a48d3702bccd3e8e9f0d611e5cb2449b369de98b9d04d56876 +size 7018235 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..989689c31c7ef4ed1f93a752f1f8011c9d3624a5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906373632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113460826112, + "active_bytes_peak_delta": 30838050304, + "after": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 113460826112, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 113460826112, + "device": 3, + "inactive_split_bytes_current": 5926594560, + "inactive_split_bytes_peak": 14129433600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2841, + "segment_count_peak": 2841, + "timestamp_utc": "2026-08-26T19:17:37.348664Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906373632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113460826112, + "allocated_bytes_peak_delta": 30838050304, + "before": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 82622775808, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 82622775808, + "device": 3, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_peak": 14129433600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 83022053376, + "reserved_bytes_peak": 83022053376, + "segment_count_current": 2127, + "segment_count_peak": 2127, + "timestamp_utc": "2026-08-26T19:17:36.066185Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5926594560, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14129433600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113862770688, + "reserved_bytes_current_delta": 30840717312, + "reserved_bytes_peak": 113862770688, + "reserved_bytes_peak_delta": 30840717312, + "segment_count_current": 2841, + "segment_count_current_delta": 714, + "segment_count_peak": 2841, + "segment_count_peak_delta": 714, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.477356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba5235fb90a56d63810a6028a9cd2c59e4812d0f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b66639ce8538a0bb11755f6ccfcdb7faa6fabbb3ce191fd07e1245df4e09489c +size 6806262 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aaca572ccab6d7f28b162ce511422b5ebbe00efe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906373632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113474031616, + "active_bytes_peak_delta": 13205504, + "after": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_peak": 14129433600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2841, + "segment_count_peak": 2841, + "timestamp_utc": "2026-08-26T19:17:38.517239Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906373632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113474031616, + "allocated_bytes_peak_delta": 13205504, + "before": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 113460826112, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 113460826112, + "device": 3, + "inactive_split_bytes_current": 5926594560, + "inactive_split_bytes_peak": 14129433600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2841, + "segment_count_peak": 2841, + "timestamp_utc": "2026-08-26T19:17:37.492920Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14129433600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113862770688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113862770688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2841, + "segment_count_current_delta": 0, + "segment_count_peak": 2841, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.645475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..898d5ace1b32ce283edbb96d04a6a88bb6934261 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8efaadcf1709cdc1b13914564edcdc3e5346f5010e53e50c204a57c22acdbd1 +size 6771814 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f51571332ca6137c5e83950a200bda6a4f96edbc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75923417088, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113474031616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75923417088, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75923417088, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_peak": 15479161344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119059513344, + "reserved_bytes_peak": 119059513344, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:17:41.821350Z", + "world_size": 8 + }, + "allocated_bytes_current": 75923417088, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113474031616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75906373632, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75906373632, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5924497408, + "inactive_split_bytes_peak": 14129433600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2841, + "segment_count_peak": 2841, + "timestamp_utc": "2026-08-26T19:17:38.660609Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15479161344, + "inactive_split_bytes_peak_delta": 1349727744, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119059513344, + "reserved_bytes_current_delta": 5196742656, + "reserved_bytes_peak": 119059513344, + "reserved_bytes_peak_delta": 5196742656, + "segment_count_current": 3136, + "segment_count_current_delta": 295, + "segment_count_peak": 3136, + "segment_count_peak_delta": 295, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.963989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11244dad4d5dc643a4a90867665f699ce7331e58 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cec3c71cc3c2b7963ee074aebfd39e9437869d8a12b22312c2c2b052a35902bc +size 7466230 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a19e70bfde29f3ab1dd62fdaffda148f11a8a1b4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75923417088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113474031616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75923417088, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75923417088, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_peak": 15479161344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119059513344, + "reserved_bytes_peak": 119059513344, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:17:42.351569Z", + "world_size": 8 + }, + "allocated_bytes_current": 75923417088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113474031616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75923417088, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75923417088, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_peak": 15479161344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119059513344, + "reserved_bytes_peak": 119059513344, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:17:41.980683Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15479161344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119059513344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119059513344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3136, + "segment_count_current_delta": 0, + "segment_count_peak": 3136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcaebe26ba46d42dbcbe1bdb7d76baf5d7b5f903 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8c8275abfb2f2937daa8e5310c30653f639027b80d6cd792edc076f091751a4 +size 901220 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6bc9b21b376f6afdc9633b41e529c365a8a2cc4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75923417088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113474031616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75923417088, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75923417088, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_peak": 15479161344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119059513344, + "reserved_bytes_peak": 119059513344, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:17:42.732501Z", + "world_size": 8 + }, + "allocated_bytes_current": 75923417088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113474031616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75923417088, + "active_bytes_peak": 113474031616, + "allocated_bytes_current": 75923417088, + "allocated_bytes_peak": 113474031616, + "device": 3, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_peak": 15479161344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119059513344, + "reserved_bytes_peak": 119059513344, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:17:42.374142Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5961979904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15479161344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119059513344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119059513344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3136, + "segment_count_current_delta": 0, + "segment_count_peak": 3136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751336Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e03c78da76c061c827484b8d4c7ed866529e4f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70cf0db8ddacad01116ec4c48b801841bbb386f001903ce288f1f9829c621bff +size 901220 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json new file mode 100644 index 0000000000000000000000000000000000000000..586c142206f051f52ae636be00dea18362c69173 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75889472000, + "active_bytes_current_delta": 32338143232, + "active_bytes_peak": 82618485248, + "active_bytes_peak_delta": 39060865024, + "after": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 82618485248, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 82618485248, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 83001081856, + "reserved_bytes_peak": 83001081856, + "segment_count_current": 2127, + "segment_count_peak": 2127, + "timestamp_utc": "2026-08-26T19:17:35.910248Z", + "world_size": 8 + }, + "allocated_bytes_current": 75889472000, + "allocated_bytes_current_delta": 32338143232, + "allocated_bytes_peak": 82618485248, + "allocated_bytes_peak_delta": 39060865024, + "before": { + "active_bytes_current": 43551328768, + "active_bytes_peak": 43557620224, + "allocated_bytes_current": 43551328768, + "allocated_bytes_peak": 43557620224, + "device": 4, + "inactive_split_bytes_current": 1254323712, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T19:17:09.289064Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_current_delta": 4649326592, + "inactive_split_bytes_peak": 14135910400, + "inactive_split_bytes_peak_delta": 12523649024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 83001081856, + "reserved_bytes_current_delta": 38180749312, + "reserved_bytes_peak": 83001081856, + "reserved_bytes_peak_delta": 38180749312, + "segment_count_current": 2127, + "segment_count_current_delta": 1238, + "segment_count_peak": 2127, + "segment_count_peak_delta": 1238, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.047700Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd8f6b8e0a5bb7745de8e7feba32fd4076b4d590 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:938c759b9247f4d8c27dc09ea87f60048674572c7c241818b3c846af4811b62d +size 7019264 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bb12c2622c0dabdf2b7d6e8018b1ecac0f6cdaa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75889472000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113449531904, + "active_bytes_peak_delta": 30831046656, + "after": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 113449531904, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 113449531904, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113894227968, + "reserved_bytes_peak": 113894227968, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-26T19:17:37.348755Z", + "world_size": 8 + }, + "allocated_bytes_current": 75889472000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113449531904, + "allocated_bytes_peak_delta": 30831046656, + "before": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 82618485248, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 82618485248, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 83001081856, + "reserved_bytes_peak": 83001081856, + "segment_count_current": 2127, + "segment_count_peak": 2127, + "timestamp_utc": "2026-08-26T19:17:36.068725Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14135910400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113894227968, + "reserved_bytes_current_delta": 30893146112, + "reserved_bytes_peak": 113894227968, + "reserved_bytes_peak_delta": 30893146112, + "segment_count_current": 2840, + "segment_count_current_delta": 713, + "segment_count_peak": 2840, + "segment_count_peak_delta": 713, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.478575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49d08ffadc44f955da2e3d36dbb77008a6f46d35 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a59db31fcb1178c6c17f0052dcce04b3b1b625978b1186d12783ae15ce5696 +size 6887605 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5d3b18cb769caef35c185cde0ae968859bdef79 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75889472000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113484609024, + "active_bytes_peak_delta": 35077120, + "after": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113894227968, + "reserved_bytes_peak": 113894227968, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-26T19:17:38.517318Z", + "world_size": 8 + }, + "allocated_bytes_current": 75889472000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113484609024, + "allocated_bytes_peak_delta": 35077120, + "before": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 113449531904, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 113449531904, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113894227968, + "reserved_bytes_peak": 113894227968, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-26T19:17:37.494110Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14135910400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113894227968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113894227968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2840, + "segment_count_current_delta": 0, + "segment_count_peak": 2840, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.650019Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c1bddf450eef45afa7c974423b5c6de8f899944 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b915e4bfb453cd576b0d714de462afc17aeec39ebeec84f050668f313c22dcc9 +size 6853737 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b53642b8cbea6033df622a4352670adcbb94a68 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906515456, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113484609024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75906515456, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75906515456, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_peak": 15458314240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119021764608, + "reserved_bytes_peak": 119021764608, + "segment_count_current": 3128, + "segment_count_peak": 3128, + "timestamp_utc": "2026-08-26T19:17:41.821352Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906515456, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113484609024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75889472000, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75889472000, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5903650304, + "inactive_split_bytes_peak": 14135910400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113894227968, + "reserved_bytes_peak": 113894227968, + "segment_count_current": 2840, + "segment_count_peak": 2840, + "timestamp_utc": "2026-08-26T19:17:38.664950Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15458314240, + "inactive_split_bytes_peak_delta": 1322403840, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119021764608, + "reserved_bytes_current_delta": 5127536640, + "reserved_bytes_peak": 119021764608, + "reserved_bytes_peak_delta": 5127536640, + "segment_count_current": 3128, + "segment_count_current_delta": 288, + "segment_count_peak": 3128, + "segment_count_peak_delta": 288, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.966789Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..237daf7e57ee3458cdabc664500bf1037f9a78d6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dad172e0d2c6dc1c5c2a863553b608c87e1edb0f663f8f11233e102566d2f0f +size 7509462 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a82e2019ba0ac6499c65deacacf8e0c467d2159 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906515456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113484609024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75906515456, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75906515456, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_peak": 15458314240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119021764608, + "reserved_bytes_peak": 119021764608, + "segment_count_current": 3128, + "segment_count_peak": 3128, + "timestamp_utc": "2026-08-26T19:17:42.351542Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906515456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113484609024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75906515456, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75906515456, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_peak": 15458314240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119021764608, + "reserved_bytes_peak": 119021764608, + "segment_count_current": 3128, + "segment_count_peak": 3128, + "timestamp_utc": "2026-08-26T19:17:41.987871Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15458314240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119021764608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119021764608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3128, + "segment_count_current_delta": 0, + "segment_count_peak": 3128, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07d44dc6c8f6ae544848da73bfe7be4f09d4ef47 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc88224475f580c8895dcd920a7e741377540aa089ea53514dc70213fe34d75 +size 903196 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d7d9c26d181e1cdb14e57111ac1b78c67c7e4f6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75906515456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113484609024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75906515456, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75906515456, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_peak": 15458314240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119021764608, + "reserved_bytes_peak": 119021764608, + "segment_count_current": 3128, + "segment_count_peak": 3128, + "timestamp_utc": "2026-08-26T19:17:42.732549Z", + "world_size": 8 + }, + "allocated_bytes_current": 75906515456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113484609024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75906515456, + "active_bytes_peak": 113484609024, + "allocated_bytes_current": 75906515456, + "allocated_bytes_peak": 113484609024, + "device": 4, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_peak": 15458314240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119021764608, + "reserved_bytes_peak": 119021764608, + "segment_count_current": 3128, + "segment_count_peak": 3128, + "timestamp_utc": "2026-08-26T19:17:42.374192Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5941132800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15458314240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119021764608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119021764608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3128, + "segment_count_current_delta": 0, + "segment_count_peak": 3128, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751576Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbd85fa64e20d314bf9330c7793f3a538b68cd6e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:860eb9dcd108badc7a929a9055d4932e004bca7ae75c76ec1aaf6dc2dd7e394f +size 903196 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f299a21eb9bd5818d7ae19715ad7e938383928cc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 32328931328, + "active_bytes_peak": 82624266240, + "active_bytes_peak_delta": 39063780352, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:35.910208Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 32328931328, + "allocated_bytes_peak": 82624266240, + "allocated_bytes_peak_delta": 39063780352, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 5, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:17:09.289231Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 4645955584, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 12537032192, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 82996887552, + "reserved_bytes_current_delta": 38161874944, + "reserved_bytes_peak": 82996887552, + "reserved_bytes_peak_delta": 38161874944, + "segment_count_current": 2112, + "segment_count_current_delta": 1229, + "segment_count_peak": 2112, + "segment_count_peak_delta": 1229, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.046760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cae3febec87cbf96de6b0d5b12d7e6f39c8c9c4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1873ea447f9791d697e95e1b4efbbaf57a09fbc11a968d7d1450883233f61756 +size 7001386 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5eb8a49449cf655207cf8b456d834189d835f2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113455617536, + "active_bytes_peak_delta": 30831351296, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.348746Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113455617536, + "allocated_bytes_peak_delta": 30831351296, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:36.066321Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 30844911616, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 30844911616, + "segment_count_current": 2812, + "segment_count_current_delta": 700, + "segment_count_peak": 2812, + "segment_count_peak_delta": 700, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.477811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5077be18f0fc9506ecbb4127cc84c3b52391db45 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bd4983fbce8115d5dfac2488b6a014fa7ba53a90610645caf927c4081abcbe3 +size 6492481 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16cd88157807cb23d2de05ee576e695092da0c8a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 43040768, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.517271Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 43040768, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.493849Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2812, + "segment_count_current_delta": 0, + "segment_count_peak": 2812, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.646742Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e35e15e7a1fe1bf7b6b6ec1ea73c90696de12e26 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191738517690Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422c04e5b708d863453debfd9380d1bf992a6698fa13548381f7e8bc53eeedf6 +size 6457972 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c29bfb932bf6e9916ea7afce7aa79a49803d015 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.821359Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.661609Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 1306978304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 5171576832, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 5171576832, + "segment_count_current": 3111, + "segment_count_current_delta": 299, + "segment_count_peak": 3111, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.961311Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3e101c306c78bb32a458b9ac33aa5314aa21bf8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191741821847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03607f6120c789b94f86ca8440180a70943185baa398900e071c1576ccdd5ed2 +size 7491889 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..33871bcb9aa95f7e4959714623de75a1bcf1afe4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.351510Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.978097Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.371928Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1b59d6ac689580a939b16c47dd22c7ebd4c7313 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191742351945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e80b811c178a1ac15f1895c2165f0fc535b2ec746f334124899ed49b463e9c2c +size 886235 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b80b7fab0b84593e8c9d3fd111fe055dbfbe0053 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.732497Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 5, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.373534Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..203dc2f47c87f6e5544321a6f851c44e61fdd653 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191742732902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8f1804085f177b692d66504875fa6c5ce00d5c5c2b008b08b5ea0d1182feee +size 886235 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f8fd93e6f1811bcf1c0625f171e792c3dbd8a86 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896428032, + "active_bytes_current_delta": 32342366208, + "active_bytes_peak": 82623743488, + "active_bytes_peak_delta": 39063390208, + "after": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 82623743488, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 82623743488, + "device": 6, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_peak": 13956707328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 83013664768, + "reserved_bytes_peak": 83013664768, + "segment_count_current": 2132, + "segment_count_peak": 2132, + "timestamp_utc": "2026-08-26T19:17:35.910183Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896428032, + "allocated_bytes_current_delta": 32342366208, + "allocated_bytes_peak": 82623743488, + "allocated_bytes_peak_delta": 39063390208, + "before": { + "active_bytes_current": 43554061824, + "active_bytes_peak": 43560353280, + "allocated_bytes_current": 43554061824, + "allocated_bytes_peak": 43560353280, + "device": 6, + "inactive_split_bytes_current": 1255784960, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:17:09.289226Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_current_delta": 4670269440, + "inactive_split_bytes_peak": 13956707328, + "inactive_split_bytes_peak_delta": 12344445952, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 83013664768, + "reserved_bytes_current_delta": 38189137920, + "reserved_bytes_peak": 83013664768, + "reserved_bytes_peak_delta": 38189137920, + "segment_count_current": 2132, + "segment_count_current_delta": 1242, + "segment_count_peak": 2132, + "segment_count_peak_delta": 1242, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.047725Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a16996d09e9e97caf63cb1edc2b3f38fff4c0a93 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191735910676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721d3d49648e52c3cfbbd2fe82ea9ed92a9432fd8d42e2f43b92368a84665435 +size 7019765 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..180e6aa6c45056f74cf43bdf4d2c027dc4308024 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896428032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113445838336, + "active_bytes_peak_delta": 30822094848, + "after": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 113445838336, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 113445838336, + "device": 6, + "inactive_split_bytes_current": 5928151552, + "inactive_split_bytes_peak": 13956707328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113833410560, + "reserved_bytes_peak": 113833410560, + "segment_count_current": 2845, + "segment_count_peak": 2845, + "timestamp_utc": "2026-08-26T19:17:37.348677Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896428032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113445838336, + "allocated_bytes_peak_delta": 30822094848, + "before": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 82623743488, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 82623743488, + "device": 6, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_peak": 13956707328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 83013664768, + "reserved_bytes_peak": 83013664768, + "segment_count_current": 2132, + "segment_count_peak": 2132, + "timestamp_utc": "2026-08-26T19:17:36.068251Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5928151552, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 13956707328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113833410560, + "reserved_bytes_current_delta": 30819745792, + "reserved_bytes_peak": 113833410560, + "reserved_bytes_peak_delta": 30819745792, + "segment_count_current": 2845, + "segment_count_current_delta": 713, + "segment_count_peak": 2845, + "segment_count_peak_delta": 713, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.479328Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca775882b2bb331304875fb193d261919094eac7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191737349094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fadc403b2f5465574beced61b72bfb53d9ede5628b85856520402150e4d3319 +size 6848699 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c5a539d084d177172899720c0ccafe85da3cdca --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896428032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113485979136, + "active_bytes_peak_delta": 40140800, + "after": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_peak": 13956707328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113833410560, + "reserved_bytes_peak": 113833410560, + "segment_count_current": 2845, + "segment_count_peak": 2845, + "timestamp_utc": "2026-08-26T19:17:38.517228Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896428032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113485979136, + "allocated_bytes_peak_delta": 40140800, + "before": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 113445838336, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 113445838336, + "device": 6, + "inactive_split_bytes_current": 5928151552, + "inactive_split_bytes_peak": 13956707328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113833410560, + "reserved_bytes_peak": 113833410560, + "segment_count_current": 2845, + "segment_count_peak": 2845, + "timestamp_utc": "2026-08-26T19:17:37.495258Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 13956707328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113833410560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113833410560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2845, + "segment_count_current_delta": 0, + "segment_count_peak": 2845, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.647146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7299eb07f663a6b27be60d3d7fe559a975b7f858 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191738517665Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9132bd78cf2a99b7720e295dfcca110185446298a355244970acbeff8819fb64 +size 6814498 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..157072dc8f31a9e07bee943261147867289ac078 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913471488, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113485979136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913471488, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75913471488, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_peak": 15480718336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119051124736, + "reserved_bytes_peak": 119051124736, + "segment_count_current": 3141, + "segment_count_peak": 3141, + "timestamp_utc": "2026-08-26T19:17:41.821331Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913471488, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113485979136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896428032, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75896428032, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5926054400, + "inactive_split_bytes_peak": 13956707328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113833410560, + "reserved_bytes_peak": 113833410560, + "segment_count_current": 2845, + "segment_count_peak": 2845, + "timestamp_utc": "2026-08-26T19:17:38.663225Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15480718336, + "inactive_split_bytes_peak_delta": 1524011008, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119051124736, + "reserved_bytes_current_delta": 5217714176, + "reserved_bytes_peak": 119051124736, + "reserved_bytes_peak_delta": 5217714176, + "segment_count_current": 3141, + "segment_count_current_delta": 296, + "segment_count_peak": 3141, + "segment_count_peak_delta": 296, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.963317Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..782dcb90cf1ee4219f79bc917b1529fea09448d4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191741821826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbccee5f7aff3bf904bdf57039d7bf4cec28a80f034143a905ffc5f7bcd05169 +size 7509746 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfd891be714ac412ae3a5668103e00e6e71bc2e2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913471488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113485979136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913471488, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75913471488, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_peak": 15480718336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119051124736, + "reserved_bytes_peak": 119051124736, + "segment_count_current": 3141, + "segment_count_peak": 3141, + "timestamp_utc": "2026-08-26T19:17:42.351513Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913471488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113485979136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913471488, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75913471488, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_peak": 15480718336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119051124736, + "reserved_bytes_peak": 119051124736, + "segment_count_current": 3141, + "segment_count_peak": 3141, + "timestamp_utc": "2026-08-26T19:17:41.980868Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15480718336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119051124736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119051124736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3141, + "segment_count_current_delta": 0, + "segment_count_peak": 3141, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372618Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe6846b327d4c81fbb3015f6dba87cbff7710c49 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191742351966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8a8070c38d7c60e3a1f8e11bbfae17d1a66cecf3f75e864c74e5b091ca728c +size 904063 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7af5ab9de7da66083e1faf4ef14c87db9002e051 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913471488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113485979136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913471488, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75913471488, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_peak": 15480718336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119051124736, + "reserved_bytes_peak": 119051124736, + "segment_count_current": 3141, + "segment_count_peak": 3141, + "timestamp_utc": "2026-08-26T19:17:42.732486Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913471488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113485979136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913471488, + "active_bytes_peak": 113485979136, + "allocated_bytes_current": 75913471488, + "allocated_bytes_peak": 113485979136, + "device": 6, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_peak": 15480718336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119051124736, + "reserved_bytes_peak": 119051124736, + "segment_count_current": 3141, + "segment_count_peak": 3141, + "timestamp_utc": "2026-08-26T19:17:42.374258Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5963536896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15480718336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119051124736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119051124736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3141, + "segment_count_current_delta": 0, + "segment_count_peak": 3141, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0889d2076d0959d3589dfa0d8cee80f5b45fe13b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191742732877Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d86acc71b55fe0208319604193d94af1acbd450da5ae44484f3906cd1a1bfe3e +size 904063 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47c1f634efcfadfe78e7e2d7677266421c08b5f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 32328931328, + "active_bytes_peak": 82624266240, + "active_bytes_peak_delta": 39063780352, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:35.910168Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 32328931328, + "allocated_bytes_peak": 82624266240, + "allocated_bytes_peak_delta": 39063780352, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 7, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:17:09.289468Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 4645955584, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 12537032192, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 82996887552, + "reserved_bytes_current_delta": 38161874944, + "reserved_bytes_peak": 82996887552, + "reserved_bytes_peak_delta": 38161874944, + "segment_count_current": 2112, + "segment_count_current_delta": 1229, + "segment_count_peak": 2112, + "segment_count_peak_delta": 1229, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.044772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdcf30d91a099e06bdb7ca8210df9b0694511ceb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191735910659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecac8fe0840229585161598cc13f8c89bfcf7e84288104dcd7f73afbc12b951f +size 7001386 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81bfe0908249a26dd658723d731aa5cb6474f73c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113455617536, + "active_bytes_peak_delta": 30831351296, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.348685Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113455617536, + "allocated_bytes_peak_delta": 30831351296, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 82624266240, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 82624266240, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 82996887552, + "reserved_bytes_peak": 82996887552, + "segment_count_current": 2112, + "segment_count_peak": 2112, + "timestamp_utc": "2026-08-26T19:17:36.066350Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 30844911616, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 30844911616, + "segment_count_current": 2812, + "segment_count_current_delta": 700, + "segment_count_peak": 2812, + "segment_count_peak_delta": 700, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.477424Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eeaac1d541a8e36146ca02de107bc8c712ee8e45 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191737349100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0c0d44c073fc75b8a8a6165a2c5c2894eb96551682994fe19ddd6ac7a405d8 +size 6492475 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b93466fdb8a98bbc06b1cfae134417a79d8da1a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75883125760, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 43040768, + "after": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.517246Z", + "world_size": 8 + }, + "allocated_bytes_current": 75883125760, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 43040768, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113455617536, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113455617536, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:37.493689Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14149293568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113841799168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113841799168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2812, + "segment_count_current_delta": 0, + "segment_count_peak": 2812, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.647838Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1ac57feb2a22ba9e9c5e0f15f897450d32ba070 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191738517674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13cf6c0c747aebec34ae027061e77c9a5103d1fd6228ba0e5301e7baadd3051f +size 6457972 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.json new file mode 100644 index 0000000000000000000000000000000000000000..275eabe10b544e8e90541812c6764ab35b53da98 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.821375Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75883125760, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75883125760, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5901607936, + "inactive_split_bytes_peak": 14149293568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113841799168, + "reserved_bytes_peak": 113841799168, + "segment_count_current": 2812, + "segment_count_peak": 2812, + "timestamp_utc": "2026-08-26T19:17:38.663549Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 1306978304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 5171576832, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 5171576832, + "segment_count_current": 3111, + "segment_count_current_delta": 299, + "segment_count_peak": 3111, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.963496Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2aa5ed6eca1e54268b391d056308c6f6e11f18b9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191741821861Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa75e0eb3dba6358a98a7cc859743ca38b837ac37e55ec418ea5c84645776a88 +size 7491889 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.json new file mode 100644 index 0000000000000000000000000000000000000000..948bfc860d0941c20be1c03e406b8a84f75b5287 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.351470Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:41.980894Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08da91f9b4d7d29f12b344b6e45da051950357bd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191742351908Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a179f158f1ee7aafba3bdffa7b7402a5d7658b3ed10deb24d1e3d4df702f19bf +size 886235 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7bd716a1098772fdb6dbc6752ae6bc041857440 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75900169216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113498658304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.732462Z", + "world_size": 8 + }, + "allocated_bytes_current": 75900169216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113498658304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75900169216, + "active_bytes_peak": 113498658304, + "allocated_bytes_current": 75900169216, + "allocated_bytes_peak": 113498658304, + "device": 7, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_peak": 15456271872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119013376000, + "reserved_bytes_peak": 119013376000, + "segment_count_current": 3111, + "segment_count_peak": 3111, + "timestamp_utc": "2026-08-26T19:17:42.373634Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5939090432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15456271872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119013376000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119013376000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3111, + "segment_count_current_delta": 0, + "segment_count_peak": 3111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751481Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ad770fbcd482623249d8abcfc5c6d687c70f0c6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191742732848Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:333b35a6494d8e0a8f8c35fea41743c479fe3691b74dc36ab407d40b893912a2 +size 886235 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..f502aefa1b7c2e566129ecf5da1f07cbc931ef1e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..de25c141cd46b71639dd48336d86e6300489fbca --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,2875 @@ +[19:16:39] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:16:40.414000 642801 torch/distributed/run.py:766] +W0826 19:16:40.414000 642801 torch/distributed/run.py:766] ***************************************** +W0826 19:16:40.414000 642801 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:16:40.414000 642801 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 76 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 19:16:58.587163879 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:16:58.587223088 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:16:58.587252839 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:16:58.587312921 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:16:58.587312473 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:16:58.587335291 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:16:58.588632455 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:16:58.738755733 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.725 seconds +time to initialize megatron (seconds): 6.640 +[after megatron is initialized] datetime: 2026-08-26 19:17:04 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_proj.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.k_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:17:09 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:17:09 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4757.10, 4784.86) + train/valid/test-data-iterators-setup ..........: (70.82, 83.22) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:17:09 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:17:35] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 26623.2 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095286E+01 | loss scale: 1.0 | grad norm: 54.362 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72374.47900390625 | max allocated: 78800.18701171875 | reserved: 79150.0 | max reserved: 79150.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:17:37] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1438.8 | throughput per GPU (TFLOP/s/GPU): 61.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095473E+01 | loss scale: 1.0 | grad norm: 56.504 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:17:38] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1168.5 | throughput per GPU (TFLOP/s/GPU): 76.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008629E+01 | loss scale: 1.0 | grad norm: 180.018 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:17:41] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 3304.1 | throughput per GPU (TFLOP/s/GPU): 27.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604090E+00 | loss scale: 1.0 | grad norm: 8.724 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:17:42] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 530.3 | throughput per GPU (TFLOP/s/GPU): 168.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307719E+00 | loss scale: 1.0 | grad norm: 4.401 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:17:42] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 381.0 | throughput per GPU (TFLOP/s/GPU): 233.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.260234E+00 | loss scale: 1.0 | grad norm: 6.098 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +[after training is done] datetime: 2026-08-26 19:17:42 +[rank0]:[W826 19:17:43.995419712 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:17:45.536468722 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:17:45.536486435 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:17:45.538748499 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:17:45.540445274 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:17:45.540877575 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:17:45.540897642 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:17:45.542531614 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:17:49.094000 642801 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:17:49.095000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642892 closing signal SIGTERM +W0826 19:17:49.095000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642893 closing signal SIGTERM +W0826 19:17:49.095000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642894 closing signal SIGTERM +W0826 19:17:49.097000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642895 closing signal SIGTERM +W0826 19:17:49.098000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642896 closing signal SIGTERM +W0826 19:17:49.099000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642897 closing signal SIGTERM +W0826 19:17:49.100000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642898 closing signal SIGTERM +W0826 19:17:49.100000 642801 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642899 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 642801 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..f3f764fb24e8d3ad2f2b127880eb4fcd328656b3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 821 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2663 +1, 2663 +2, 2663 +3, 2663 +4, 2663 +5, 2663 +6, 2663 +7, 2663 +0, 41187 +1, 41187 +2, 41187 +3, 41187 +4, 41187 +5, 41187 +6, 41187 +7, 41187 +0, 47061 +1, 47103 +2, 47133 +3, 47077 +4, 47069 +5, 47065 +6, 47045 +7, 47077 +0, 48477 +1, 48479 +2, 48489 +3, 48451 +4, 48475 +5, 48489 +6, 48479 +7, 48489 +0, 48647 +1, 48649 +2, 48659 +3, 48621 +4, 48645 +5, 48659 +6, 48649 +7, 48659 +0, 48807 +1, 48809 +2, 48819 +3, 48781 +4, 48805 +5, 48819 +6, 48809 +7, 48819 +0, 49511 +1, 49513 +2, 49515 +3, 49485 +4, 49485 +5, 49515 +6, 49469 +7, 49515 +0, 57275 +1, 57359 +2, 57377 +3, 57289 +4, 57371 +5, 57293 +6, 57315 +7, 57377 +0, 86271 +1, 86275 +2, 86283 +3, 86297 +4, 86283 +5, 86273 +6, 86263 +7, 86279 +0, 86689 +1, 86691 +2, 86707 +3, 86703 +4, 86703 +5, 86707 +6, 86687 +7, 86707 +0, 86689 +1, 86691 +2, 86707 +3, 86703 +4, 86703 +5, 86707 +6, 86687 +7, 86707 +0, 86953 +1, 86973 +2, 87013 +3, 86977 +4, 87007 +5, 86997 +6, 86955 +7, 87003 +0, 87737 +1, 87739 +2, 87755 +3, 87751 +4, 87751 +5, 87755 +6, 87735 +7, 87755 +0, 88359 +1, 88383 +2, 88377 +3, 88411 +4, 88369 +5, 88377 +6, 88417 +7, 88379 +0, 117877 +1, 117855 +2, 117895 +3, 117851 +4, 117907 +5, 117895 +6, 117855 +7, 117895 +0, 100769 +1, 102625 +2, 101855 +3, 100881 +4, 101069 +5, 102083 +6, 101275 +7, 101369 +0, 123729 +1, 124053 +2, 123573 +3, 123739 +4, 123745 +5, 123573 +6, 123725 +7, 123573 +0, 122913 +1, 123237 +2, 122757 +3, 122923 +4, 122929 +5, 122757 +6, 122909 +7, 122757 +0, 119691 +1, 120015 +2, 119535 +3, 119701 +4, 119707 +5, 119535 +6, 120075 +7, 119535 +0, 117623 +1, 118335 +2, 117855 +3, 117733 +4, 118027 +5, 117467 +6, 118007 +7, 117649 +0, 1 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 117649 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a63b0e7580f2a0a4e0112fc278d63586fc61e61 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881990144, + "active_bytes_current_delta": 32329736704, + "active_bytes_peak": 83807716352, + "active_bytes_peak_delta": 40249171456, + "after": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 83807716352, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 83807716352, + "device": 0, + "inactive_split_bytes_current": 6722729984, + "inactive_split_bytes_peak": 12798374912, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 84232110080, + "reserved_bytes_peak": 84232110080, + "segment_count_current": 2132, + "segment_count_peak": 2132, + "timestamp_utc": "2026-08-26T19:18:51.629392Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881990144, + "allocated_bytes_current_delta": 32329736704, + "allocated_bytes_peak": 83807716352, + "allocated_bytes_peak_delta": 40249171456, + "before": { + "active_bytes_current": 43552253440, + "active_bytes_peak": 43558544896, + "allocated_bytes_current": 43552253440, + "allocated_bytes_peak": 43558544896, + "device": 0, + "inactive_split_bytes_current": 1255496192, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T19:18:24.522414Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6722729984, + "inactive_split_bytes_current_delta": 5467233792, + "inactive_split_bytes_peak": 12798374912, + "inactive_split_bytes_peak_delta": 11186113536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 84232110080, + "reserved_bytes_current_delta": 39409680384, + "reserved_bytes_peak": 84232110080, + "reserved_bytes_peak_delta": 39409680384, + "segment_count_current": 2132, + "segment_count_current_delta": 1247, + "segment_count_peak": 2132, + "segment_count_peak_delta": 1247, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.763866Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46dfba2980ebdd7d13c8a13dc9e971d48025dc12 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191851629863Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbb0077847eeee97f9feba4c4ed4af341dec26a70f4e0c103bb099530cf1b216 +size 7016925 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbf486846206d236d30fc1cb7c25e78c509f4aa7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881990144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114569909248, + "active_bytes_peak_delta": 30762192896, + "after": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_peak": 12798374912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_peak": 115020398592, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-26T19:18:52.883763Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881990144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114569909248, + "allocated_bytes_peak_delta": 30762192896, + "before": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 83807716352, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 83807716352, + "device": 0, + "inactive_split_bytes_current": 6722729984, + "inactive_split_bytes_peak": 12798374912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 84232110080, + "reserved_bytes_peak": 84232110080, + "segment_count_current": 2132, + "segment_count_peak": 2132, + "timestamp_utc": "2026-08-26T19:18:51.782720Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12798374912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_current_delta": 30788288512, + "reserved_bytes_peak": 115020398592, + "reserved_bytes_peak_delta": 30788288512, + "segment_count_current": 2766, + "segment_count_current_delta": 634, + "segment_count_peak": 2766, + "segment_count_peak_delta": 634, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.019297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..05f46288da574309d07e71850d066c4050efcccb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191852884234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08eb978a29271754e7d13464e3ef9bf769fd7e1dfddfff2a52fcc4eb22f55a16 +size 7144770 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e38b0f9403a00b2d1f3e37c675920002d0fb8fd9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881990144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114569909248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_peak": 12798374912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_peak": 115020398592, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-26T19:18:54.042083Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881990144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114569909248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_peak": 12798374912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_peak": 115020398592, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-26T19:18:53.035054Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12798374912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115020398592, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2766, + "segment_count_current_delta": 0, + "segment_count_peak": 2766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.181745Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..82c5186b1a7fe0a9cb97bfa3865469e9810db611 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191854042542Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c1f4ed079d1e4ed5fd92993d33d76351d84c5cc20f5520a6094423d6ecd391 +size 7115873 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1958051bdb88ba89fc7d821f72a981ca20f736d0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75899033600, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114569909248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75899033600, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75899033600, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_peak": 16379303936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_peak": 121079070720, + "segment_count_current": 3054, + "segment_count_peak": 3054, + "timestamp_utc": "2026-08-26T19:18:57.301277Z", + "world_size": 8 + }, + "allocated_bytes_current": 75899033600, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114569909248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75881990144, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75881990144, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6724827136, + "inactive_split_bytes_peak": 12798374912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115020398592, + "reserved_bytes_peak": 115020398592, + "segment_count_current": 2766, + "segment_count_peak": 2766, + "timestamp_utc": "2026-08-26T19:18:54.197782Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16379303936, + "inactive_split_bytes_peak_delta": 3580929024, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_current_delta": 6058672128, + "reserved_bytes_peak": 121079070720, + "reserved_bytes_peak_delta": 6058672128, + "segment_count_current": 3054, + "segment_count_current_delta": 288, + "segment_count_peak": 3054, + "segment_count_peak_delta": 288, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.445624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08cbb41c4ccdbbe7d2850b97f71d0986c7f0808e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191857301775Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c9838012b62eebd1b17f9b956af28bc9d5b139924f447a0cddfd342b0c577b1 +size 7490178 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aed654d92a319bdf5be3fee3aa69892bc103c6dc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75899033600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114569909248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75899033600, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75899033600, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_peak": 16379303936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_peak": 121079070720, + "segment_count_current": 3054, + "segment_count_peak": 3054, + "timestamp_utc": "2026-08-26T19:18:57.833111Z", + "world_size": 8 + }, + "allocated_bytes_current": 75899033600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114569909248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75899033600, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75899033600, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_peak": 16379303936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_peak": 121079070720, + "segment_count_current": 3054, + "segment_count_peak": 3054, + "timestamp_utc": "2026-08-26T19:18:57.466544Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16379303936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121079070720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3054, + "segment_count_current_delta": 0, + "segment_count_peak": 3054, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.853913Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a905ced611bea36fbb2536843967a273b9942d1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191857833550Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1189d4039ca2b3934e95ed1beac79f62a14d7fee17fad66d733665eb10b469e7 +size 890447 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a57ac9d66ee4dd22ec9e51fb12a981637ee4a824 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75899033600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114569909248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75899033600, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75899033600, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_peak": 16379303936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_peak": 121079070720, + "segment_count_current": 3054, + "segment_count_peak": 3054, + "timestamp_utc": "2026-08-26T19:18:58.216039Z", + "world_size": 8 + }, + "allocated_bytes_current": 75899033600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114569909248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75899033600, + "active_bytes_peak": 114569909248, + "allocated_bytes_current": 75899033600, + "allocated_bytes_peak": 114569909248, + "device": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_peak": 16379303936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_peak": 121079070720, + "segment_count_current": 3054, + "segment_count_peak": 3054, + "timestamp_utc": "2026-08-26T19:18:57.855521Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6825224192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16379303936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121079070720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121079070720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3054, + "segment_count_current_delta": 0, + "segment_count_peak": 3054, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234746Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..171314e556abcab284fe736cc0d5e7b061938d41 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191858216434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f24ca756e043383d8aa62892fc86ae747847f55aea93979e330a9723d7ab8cc5 +size 890447 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..172f97f23551e85ea26dea60f99e3c2f2ec82c92 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896686080, + "active_bytes_current_delta": 32342689792, + "active_bytes_peak": 83812233216, + "active_bytes_peak_delta": 40251945472, + "after": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 83812233216, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 83812233216, + "device": 1, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_peak": 13098621952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 84234207232, + "reserved_bytes_peak": 84234207232, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.629170Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896686080, + "allocated_bytes_current_delta": 32342689792, + "allocated_bytes_peak": 83812233216, + "allocated_bytes_peak_delta": 40251945472, + "before": { + "active_bytes_current": 43553996288, + "active_bytes_peak": 43560287744, + "allocated_bytes_current": 43553996288, + "allocated_bytes_peak": 43560287744, + "device": 1, + "inactive_split_bytes_current": 1255850496, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:18:24.522391Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_current_delta": 5794019328, + "inactive_split_bytes_peak": 13098621952, + "inactive_split_bytes_peak_delta": 11486360576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 84234207232, + "reserved_bytes_current_delta": 39409680384, + "reserved_bytes_peak": 84234207232, + "reserved_bytes_peak_delta": 39409680384, + "segment_count_current": 2137, + "segment_count_current_delta": 1247, + "segment_count_peak": 2137, + "segment_count_peak_delta": 1247, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.764013Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9837d6b3d5e624c7cc5b34cf1c69d9d14025a166 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191851629659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:690dd60a8932a16fd2728a7fb3b7231aba6a45d493c5310fedbbe7494afd52e3 +size 7018517 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c89a98bf911382648ca02e0783370fa7e198efd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896686080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114563579904, + "active_bytes_peak_delta": 30751346688, + "after": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7051966976, + "inactive_split_bytes_peak": 13098621952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2769, + "segment_count_peak": 2769, + "timestamp_utc": "2026-08-26T19:18:52.883637Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896686080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114563579904, + "allocated_bytes_peak_delta": 30751346688, + "before": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 83812233216, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 83812233216, + "device": 1, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_peak": 13098621952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 84234207232, + "reserved_bytes_peak": 84234207232, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.783155Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7051966976, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 13098621952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 114997329920, + "reserved_bytes_current_delta": 30763122688, + "reserved_bytes_peak": 114997329920, + "reserved_bytes_peak_delta": 30763122688, + "segment_count_current": 2769, + "segment_count_current_delta": 632, + "segment_count_peak": 2769, + "segment_count_peak_delta": 632, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.019562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3efdb874c1e445e79ccb3d383d3da402824c089e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191852884095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d8ebb4f762f1b2e96a814899ef9d7fef516047ac30d5e58421b4a3775e9266c +size 6746287 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e93bd688c20ed943a16be9c5b45411bfe8ca122a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896686080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114563579904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_peak": 13098621952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2769, + "segment_count_peak": 2769, + "timestamp_utc": "2026-08-26T19:18:54.041947Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896686080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114563579904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7051966976, + "inactive_split_bytes_peak": 13098621952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2769, + "segment_count_peak": 2769, + "timestamp_utc": "2026-08-26T19:18:53.035871Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 13098621952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 114997329920, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114997329920, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2769, + "segment_count_current_delta": 0, + "segment_count_peak": 2769, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.171363Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1ae160a1bbbc4ce8b25aafca995641925582313 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191854042418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe77570f6a95b579502238bc5754435c5f6d8d06999f3940f4063f019cc96929 +size 6716396 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a37552e60645f1ef6fd93e69e00c2ccc6ca53e5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913729536, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114563579904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913729536, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75913729536, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_peak": 16704346624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121418809344, + "reserved_bytes_peak": 121418809344, + "segment_count_current": 3062, + "segment_count_peak": 3062, + "timestamp_utc": "2026-08-26T19:18:57.301124Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913729536, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114563579904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896686080, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75896686080, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7049869824, + "inactive_split_bytes_peak": 13098621952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2769, + "segment_count_peak": 2769, + "timestamp_utc": "2026-08-26T19:18:54.186309Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_current_delta": 102494208, + "inactive_split_bytes_peak": 16704346624, + "inactive_split_bytes_peak_delta": 3605724672, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121418809344, + "reserved_bytes_current_delta": 6421479424, + "reserved_bytes_peak": 121418809344, + "reserved_bytes_peak_delta": 6421479424, + "segment_count_current": 3062, + "segment_count_current_delta": 293, + "segment_count_peak": 3062, + "segment_count_peak_delta": 293, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.440987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9842bbf22e8d27a09a83a71157d9bea0f1a745ee --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191857301628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b591a14966953b663895c857d02158d757337db3b0b4703f13c994f449902958 +size 7493602 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.json new file mode 100644 index 0000000000000000000000000000000000000000..866694aa65732714f67f287826d735e70b2421c2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913729536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114563579904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913729536, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75913729536, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_peak": 16704346624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121418809344, + "reserved_bytes_peak": 121418809344, + "segment_count_current": 3062, + "segment_count_peak": 3062, + "timestamp_utc": "2026-08-26T19:18:57.833004Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913729536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114563579904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913729536, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75913729536, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_peak": 16704346624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121418809344, + "reserved_bytes_peak": 121418809344, + "segment_count_current": 3062, + "segment_count_peak": 3062, + "timestamp_utc": "2026-08-26T19:18:57.457507Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16704346624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121418809344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121418809344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3062, + "segment_count_current_delta": 0, + "segment_count_peak": 3062, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.853225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9c72391863f03a684a70f57695a5addfd6b6f2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191857833449Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d504edb9e4417bb7cf112a58e996510bae61efd9ec5a66b8c1dc2c01b9f926 +size 893441 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.json new file mode 100644 index 0000000000000000000000000000000000000000..648edebcd7efee1bd27921458551b8b2730fa9c9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913729536, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114563579904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913729536, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75913729536, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_peak": 16704346624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121418809344, + "reserved_bytes_peak": 121418809344, + "segment_count_current": 3062, + "segment_count_peak": 3062, + "timestamp_utc": "2026-08-26T19:18:58.215882Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913729536, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114563579904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913729536, + "active_bytes_peak": 114563579904, + "allocated_bytes_current": 75913729536, + "allocated_bytes_peak": 114563579904, + "device": 1, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_peak": 16704346624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121418809344, + "reserved_bytes_peak": 121418809344, + "segment_count_current": 3062, + "segment_count_peak": 3062, + "timestamp_utc": "2026-08-26T19:18:57.854793Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7152364032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16704346624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121418809344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121418809344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3062, + "segment_count_current_delta": 0, + "segment_count_peak": 3062, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1949443b069d415227d8e6bb89d323551bcf5515 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191858216277Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72eaacf2623be9d21f260dced7fc066f1c47d14d91083ed6452a4d2224288004 +size 893441 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e2908a8c0c19ff07827e964434874e65dcdbc80 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 32342596096, + "active_bytes_peak": 83806146560, + "active_bytes_peak_delta": 40245660672, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 2, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.629143Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 32342596096, + "allocated_bytes_peak": 83806146560, + "allocated_bytes_peak_delta": 40245660672, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 2, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:18:24.522267Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_current_delta": 5288699392, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 11017812480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 84250984448, + "reserved_bytes_current_delta": 39415971840, + "reserved_bytes_peak": 84250984448, + "reserved_bytes_peak_delta": 39415971840, + "segment_count_current": 2126, + "segment_count_current_delta": 1243, + "segment_count_peak": 2126, + "segment_count_peak_delta": 1243, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.762935Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..537de5a16a03d115bfcd5d57ae3a23b5ce61abd7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191851629645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36147b31daacedfc6c44b6c2f59a35e5051554f1f30012d042538c6dab998f41 +size 7002568 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5bf088436410463f1ae6593adef76593ec9f510 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 30782408192, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:52.883632Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 30782408192, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 2, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.783622Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 30788288512, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 30788288512, + "segment_count_current": 2751, + "segment_count_current_delta": 625, + "segment_count_peak": 2751, + "segment_count_peak_delta": 625, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.015410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..928a0925c02e59a556324f6c16da92b31e2425a7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191852884085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e644d1982b4464157c5fb67bbfaa6d21b4c7f2a8e12ebc79e4db73a10213dc +size 6610721 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.json new file mode 100644 index 0000000000000000000000000000000000000000..070cbf101c923d937ce40b002a63044b1e3d89cd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.041935Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:53.030794Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.170553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4e7c28dbea6429b5c5b6547296709e167904cdf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191854042381Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ee64a013f4d0d7107e92830fc1aee72e404e434748102ca54dd0eb0b30175b5 +size 6580515 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afecacc36e24bfb655d3a051de0dbdea58793160 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.301121Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.185194Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 3570851840, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 5876219904, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 5876219904, + "segment_count_current": 3040, + "segment_count_current_delta": 289, + "segment_count_peak": 3040, + "segment_count_peak_delta": 289, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.440236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..806225c2a37a2b4bb9996c8fcd6414fbc5506095 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191857301686Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aacbf3ffbe1cff161cba662b2521b5df9855ee07988e40b5e261036cbd860d5a +size 7477265 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68370232b45c8e04de9a7a174ee70682ca88d5d9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.832946Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.456523Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.852791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f498f51cca300206b8bf877a3d2d55643a3af30e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191857833375Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20f2a0691854e4d5ed84127254e6a647e647d12dd37bac599ef4e0a8e46988c +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c2375b26cbc04735e25ceeef0f271fbca23b64e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:58.215869Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 2, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.854369Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7aef7e6acc42a2ecc5c3ad7c4fa21cc28fb6582 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191858216255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a57c42a971f07623032f66d639d7a9ce3113e26b34e123e9386303463df76ee9 +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b8ad1cc1f913b0d4cf5ec7ea3e769a618576332 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901261824, + "active_bytes_current_delta": 32343788032, + "active_bytes_peak": 83802341888, + "active_bytes_peak_delta": 40238576640, + "after": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 83802341888, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 83802341888, + "device": 3, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_peak": 12616469504, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 84246790144, + "reserved_bytes_peak": 84246790144, + "segment_count_current": 2136, + "segment_count_peak": 2136, + "timestamp_utc": "2026-08-26T19:18:51.629257Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901261824, + "allocated_bytes_current_delta": 32343788032, + "allocated_bytes_peak": 83802341888, + "allocated_bytes_peak_delta": 40238576640, + "before": { + "active_bytes_current": 43557473792, + "active_bytes_peak": 43563765248, + "allocated_bytes_current": 43557473792, + "allocated_bytes_peak": 43563765248, + "device": 3, + "inactive_split_bytes_current": 1223012864, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T19:18:24.522395Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_current_delta": 5493028352, + "inactive_split_bytes_peak": 12616469504, + "inactive_split_bytes_peak_delta": 11004208128, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 84246790144, + "reserved_bytes_current_delta": 39451623424, + "reserved_bytes_peak": 84246790144, + "reserved_bytes_peak_delta": 39451623424, + "segment_count_current": 2136, + "segment_count_current_delta": 1249, + "segment_count_peak": 2136, + "segment_count_peak_delta": 1249, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.762680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab1a58b4bbc09b314e5dcde628913da78761f9a5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191851629737Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:419741f6c33c26f88cb4602cdca598cf8cd03566ba0d849f6f06073b6dfb99a2 +size 7018827 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d23acd2ef8932015b7e95e42275b80797c935738 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901261824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114523504640, + "active_bytes_peak_delta": 30721162752, + "after": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6718138368, + "inactive_split_bytes_peak": 12616469504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114993135616, + "reserved_bytes_peak": 114993135616, + "segment_count_current": 2772, + "segment_count_peak": 2772, + "timestamp_utc": "2026-08-26T19:18:52.883801Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901261824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114523504640, + "allocated_bytes_peak_delta": 30721162752, + "before": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 83802341888, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 83802341888, + "device": 3, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_peak": 12616469504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 84246790144, + "reserved_bytes_peak": 84246790144, + "segment_count_current": 2136, + "segment_count_peak": 2136, + "timestamp_utc": "2026-08-26T19:18:51.782045Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6718138368, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12616469504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114993135616, + "reserved_bytes_current_delta": 30746345472, + "reserved_bytes_peak": 114993135616, + "reserved_bytes_peak_delta": 30746345472, + "segment_count_current": 2772, + "segment_count_current_delta": 636, + "segment_count_peak": 2772, + "segment_count_peak_delta": 636, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.018195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..04bd70e77197773d5ff8aef5ca57061f9140e9ad --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191852884275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75ef45d0273cc40ac12f7fba9e40e2761c166a7551812e62f170a7c46b14bf09 +size 7184956 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8424073bb4b62020edc962551ed4387dee9fcf6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901261824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114523504640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_peak": 12616469504, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114993135616, + "reserved_bytes_peak": 114993135616, + "segment_count_current": 2772, + "segment_count_peak": 2772, + "timestamp_utc": "2026-08-26T19:18:54.042082Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901261824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114523504640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6718138368, + "inactive_split_bytes_peak": 12616469504, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114993135616, + "reserved_bytes_peak": 114993135616, + "segment_count_current": 2772, + "segment_count_peak": 2772, + "timestamp_utc": "2026-08-26T19:18:53.034366Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 12616469504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114993135616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114993135616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2772, + "segment_count_current_delta": 0, + "segment_count_peak": 2772, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.173294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..021776928f2466dcf967dd61dcc3d30df63cef3e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191854042638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:145e9d2182ad1f258e80e2cf7994c1ffb69e63084350f9c4c5eb89f0658653d5 +size 7014487 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3cc1086d12f17062f1b0a7ea229caee7eed2279 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918305280, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114523504640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918305280, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75918305280, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_peak": 16370518016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 121089556480, + "reserved_bytes_peak": 121089556480, + "segment_count_current": 3056, + "segment_count_peak": 3056, + "timestamp_utc": "2026-08-26T19:18:57.301253Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918305280, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114523504640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75901261824, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75901261824, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6716041216, + "inactive_split_bytes_peak": 12616469504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114993135616, + "reserved_bytes_peak": 114993135616, + "segment_count_current": 2772, + "segment_count_peak": 2772, + "timestamp_utc": "2026-08-26T19:18:54.188992Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_current_delta": 102494208, + "inactive_split_bytes_peak": 16370518016, + "inactive_split_bytes_peak_delta": 3754048512, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 121089556480, + "reserved_bytes_current_delta": 6096420864, + "reserved_bytes_peak": 121089556480, + "reserved_bytes_peak_delta": 6096420864, + "segment_count_current": 3056, + "segment_count_current_delta": 284, + "segment_count_peak": 3056, + "segment_count_peak_delta": 284, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.441230Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf422e1ffeb8c8cf3034690f3dc585ff4fb61ea1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191857301741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4cd072f2fb81b7658f39aa849f5d6798f8ff9b56db9f5046cc4363ad249d090 +size 7492989 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36f93653ffa4d151329c79486d2a244ef7ae95a9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918305280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114523504640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918305280, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75918305280, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_peak": 16370518016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 121089556480, + "reserved_bytes_peak": 121089556480, + "segment_count_current": 3056, + "segment_count_peak": 3056, + "timestamp_utc": "2026-08-26T19:18:57.833092Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918305280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114523504640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75918305280, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75918305280, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_peak": 16370518016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 121089556480, + "reserved_bytes_peak": 121089556480, + "segment_count_current": 3056, + "segment_count_peak": 3056, + "timestamp_utc": "2026-08-26T19:18:57.457958Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16370518016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 121089556480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121089556480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3056, + "segment_count_current_delta": 0, + "segment_count_peak": 3056, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.853932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba6b5d51a96c8334d9d3819cbc553d62087711da --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191857833533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fe2a3c95982fcd94ffa918a14874b73a5eece8e2526eba1a1c02dc442c0e82a +size 893028 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8fe77e2de3cbac1f7cf35be0015dd2e904c7a571 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75918305280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114523504640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75918305280, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75918305280, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_peak": 16370518016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 121089556480, + "reserved_bytes_peak": 121089556480, + "segment_count_current": 3056, + "segment_count_peak": 3056, + "timestamp_utc": "2026-08-26T19:18:58.216022Z", + "world_size": 8 + }, + "allocated_bytes_current": 75918305280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114523504640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75918305280, + "active_bytes_peak": 114523504640, + "allocated_bytes_current": 75918305280, + "allocated_bytes_peak": 114523504640, + "device": 3, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_peak": 16370518016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 121089556480, + "reserved_bytes_peak": 121089556480, + "segment_count_current": 3056, + "segment_count_peak": 3056, + "timestamp_utc": "2026-08-26T19:18:57.855552Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6818535424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16370518016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 121089556480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121089556480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3056, + "segment_count_current_delta": 0, + "segment_count_peak": 3056, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234933Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..57ac26fac6cdd96daaa8fc39d279542a822075d5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191858216411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f036755e64565f5b2366b1f9ae56d3bde18964f0c39dc31144a128e1d155fbac +size 893028 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d8d88dc7ddf144f4da836add861d7b1c93c1209 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75903358464, + "active_bytes_current_delta": 32352029696, + "active_bytes_peak": 83801721344, + "active_bytes_peak_delta": 40244101120, + "after": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 83801721344, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 83801721344, + "device": 4, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_peak": 12718891008, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 84246790144, + "reserved_bytes_peak": 84246790144, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.629251Z", + "world_size": 8 + }, + "allocated_bytes_current": 75903358464, + "allocated_bytes_current_delta": 32352029696, + "allocated_bytes_peak": 83801721344, + "allocated_bytes_peak_delta": 40244101120, + "before": { + "active_bytes_current": 43551328768, + "active_bytes_peak": 43557620224, + "allocated_bytes_current": 43551328768, + "allocated_bytes_peak": 43557620224, + "device": 4, + "inactive_split_bytes_current": 1254323712, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T19:18:24.522473Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_current_delta": 5465912320, + "inactive_split_bytes_peak": 12718891008, + "inactive_split_bytes_peak_delta": 11106629632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 84246790144, + "reserved_bytes_current_delta": 39426457600, + "reserved_bytes_peak": 84246790144, + "reserved_bytes_peak_delta": 39426457600, + "segment_count_current": 2137, + "segment_count_current_delta": 1248, + "segment_count_peak": 2137, + "segment_count_peak_delta": 1248, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.764583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..240147ca532879f11a5c6820eeecd0a0c1941d8e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191851629688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68764a07d6abc5333b37a260bed7d03150219c5dff44bb355447831cc085be23 +size 7019591 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d9746e2fb14bc2555d07b8a1b8813dbc426603c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75903358464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114603187712, + "active_bytes_peak_delta": 30801466368, + "after": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6722333184, + "inactive_split_bytes_peak": 12718891008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115051855872, + "reserved_bytes_peak": 115051855872, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:52.883700Z", + "world_size": 8 + }, + "allocated_bytes_current": 75903358464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114603187712, + "allocated_bytes_peak_delta": 30801466368, + "before": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 83801721344, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 83801721344, + "device": 4, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_peak": 12718891008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 84246790144, + "reserved_bytes_peak": 84246790144, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.783824Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6722333184, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12718891008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115051855872, + "reserved_bytes_current_delta": 30805065728, + "reserved_bytes_peak": 115051855872, + "reserved_bytes_peak_delta": 30805065728, + "segment_count_current": 2774, + "segment_count_current_delta": 637, + "segment_count_peak": 2774, + "segment_count_peak_delta": 637, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.017028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bcc304116bb2d6e591dd412d0e1695bfb709142 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191852884142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91a6e2773779fd1beb75e167737744a12ef7429cea4bcb0bfb7655318365e0df +size 6967128 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9be2ce99b83d2d31e46697a0ed5d168ca7b3d1f1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75903358464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114603187712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_peak": 12718891008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115051855872, + "reserved_bytes_peak": 115051855872, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:54.042048Z", + "world_size": 8 + }, + "allocated_bytes_current": 75903358464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114603187712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6722333184, + "inactive_split_bytes_peak": 12718891008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115051855872, + "reserved_bytes_peak": 115051855872, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:53.033222Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 12718891008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115051855872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115051855872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2774, + "segment_count_current_delta": 0, + "segment_count_peak": 2774, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.173098Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d29409b4cb13f92ca7a4b82e9a8d8c9e9a49fdf7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191854042499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58637f762420e0d892ff7935bfd35bd7a41a219c0d0ff878405b9be741c83c08 +size 6937167 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e71e01f86155c6c3b068e1eb64b6abe1684338dc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75920401920, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114603187712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75920401920, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75920401920, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_peak": 16374712832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121095847936, + "reserved_bytes_peak": 121095847936, + "segment_count_current": 3060, + "segment_count_peak": 3060, + "timestamp_utc": "2026-08-26T19:18:57.301231Z", + "world_size": 8 + }, + "allocated_bytes_current": 75920401920, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114603187712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75903358464, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75903358464, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6720236032, + "inactive_split_bytes_peak": 12718891008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115051855872, + "reserved_bytes_peak": 115051855872, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:54.188480Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_current_delta": 102494208, + "inactive_split_bytes_peak": 16374712832, + "inactive_split_bytes_peak_delta": 3655821824, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121095847936, + "reserved_bytes_current_delta": 6043992064, + "reserved_bytes_peak": 121095847936, + "reserved_bytes_peak_delta": 6043992064, + "segment_count_current": 3060, + "segment_count_current_delta": 286, + "segment_count_peak": 3060, + "segment_count_peak_delta": 286, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.444192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4b24ce2f69fddc23c840e2b075ecd848b2f8dd7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191857301732Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35c64c6f2f074c4fd2e66802036da9e5310ac4204fcb1ad05db4f3bb51fdaa2a +size 7494250 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bf5bf73f0f45b847be449085547e11f4d1921c7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75920401920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114603187712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75920401920, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75920401920, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_peak": 16374712832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121095847936, + "reserved_bytes_peak": 121095847936, + "segment_count_current": 3060, + "segment_count_peak": 3060, + "timestamp_utc": "2026-08-26T19:18:57.833054Z", + "world_size": 8 + }, + "allocated_bytes_current": 75920401920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114603187712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75920401920, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75920401920, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_peak": 16374712832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121095847936, + "reserved_bytes_peak": 121095847936, + "segment_count_current": 3060, + "segment_count_peak": 3060, + "timestamp_utc": "2026-08-26T19:18:57.461081Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16374712832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121095847936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121095847936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3060, + "segment_count_current_delta": 0, + "segment_count_peak": 3060, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.853256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f92b3a90653f2351083559827e4c72d7acd726c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191857833497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55fa372ef2b4b2034b6c270ac3ac80f9348c1dc5f8aa30d242a5f8346c7f2184 +size 894226 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7016601cae273ac61b9e781d751c24cab13a396c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75920401920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114603187712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75920401920, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75920401920, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_peak": 16374712832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121095847936, + "reserved_bytes_peak": 121095847936, + "segment_count_current": 3060, + "segment_count_peak": 3060, + "timestamp_utc": "2026-08-26T19:18:58.215967Z", + "world_size": 8 + }, + "allocated_bytes_current": 75920401920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114603187712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75920401920, + "active_bytes_peak": 114603187712, + "allocated_bytes_current": 75920401920, + "allocated_bytes_peak": 114603187712, + "device": 4, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_peak": 16374712832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121095847936, + "reserved_bytes_peak": 121095847936, + "segment_count_current": 3060, + "segment_count_peak": 3060, + "timestamp_utc": "2026-08-26T19:18:57.854849Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6822730240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16374712832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121095847936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121095847936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3060, + "segment_count_current_delta": 0, + "segment_count_peak": 3060, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b092357f1d4bce1c3dd85eb96907b1f74b8df03 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191858216359Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de376b89af91c8989a62193da92dacf3fe9dd58db0f0ad90c6bc573a09c35ac8 +size 894226 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2d943b45fe75c9eb7c0422189216b1e4213a6f0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 32342596096, + "active_bytes_peak": 83806146560, + "active_bytes_peak_delta": 40245660672, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 5, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.629156Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 32342596096, + "allocated_bytes_peak": 83806146560, + "allocated_bytes_peak_delta": 40245660672, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 5, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:18:24.522343Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_current_delta": 5288699392, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 11017812480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 84250984448, + "reserved_bytes_current_delta": 39415971840, + "reserved_bytes_peak": 84250984448, + "reserved_bytes_peak_delta": 39415971840, + "segment_count_current": 2126, + "segment_count_current_delta": 1243, + "segment_count_peak": 2126, + "segment_count_peak_delta": 1243, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.761916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed67d930c57cab97a8dfff714e9d5a7766e9ecac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191851629657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09bf6ec9617e7a305cc9226f8c6cb2a550701fdc2574ed8a667c932d4d823553 +size 7002568 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12f840911ece6a748b4a4b505b040dea2ee245f9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 30782408192, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:52.883571Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 30782408192, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 5, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.781155Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 30788288512, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 30788288512, + "segment_count_current": 2751, + "segment_count_current_delta": 625, + "segment_count_peak": 2751, + "segment_count_peak_delta": 625, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.013313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..741c425c6a67fe3dc647213bede707c9bb63714e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191852883995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8de7e77d8f9ba1dc3dd7c921cc1d184aec4115b34d0f08d5748f71f9e6a015e +size 6610725 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2911b249de5a4daa4be4b68d730c1c498abce524 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.041913Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:53.028707Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.171565Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71a92fdbd265fb9d56f6ba26959e39b5b5ee00ea --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191854042344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937806a2877639bb5e827e2b19a148d91c7841e130ffe3ddcabc4fbd201969a3 +size 6580515 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee542ca7516d3f252f3ada81e0ba3f7ff962e02f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.301118Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.187220Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 3570851840, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 5876219904, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 5876219904, + "segment_count_current": 3040, + "segment_count_current_delta": 289, + "segment_count_peak": 3040, + "segment_count_peak_delta": 289, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.442076Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cfe5146d652e505b29761d79df9712af242a438 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191857301645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76758e170ff02554574f61c001a760b4f2a5489d048a0f8c570ba21cf776b451 +size 7477265 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4982d311a8fb08a90cfeafb6a9afd0be925549ec --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.832951Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.458489Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.852971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc612662acd466713075ba103c56b5486dda63c1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191857833380Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf871dda1d7654744a2027658479a8c0d35040ee7bcaa033558d92464faa0a2 +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3efb850580a2b0fc7f5491fee079c7e42482e922 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:58.215863Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 5, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.854526Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f652e36e9c43958abddc4b368df72b3d3452b83 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191858216249Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08fcafbd1f29ff279082d4e46d92a342ad520c740afa28778f81eedfd2763015 +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ffa6b2701e71f711c505097ece129083fa46606 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75893799936, + "active_bytes_current_delta": 32339738112, + "active_bytes_peak": 83802752512, + "active_bytes_peak_delta": 40242399232, + "after": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 83802752512, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 83802752512, + "device": 6, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_peak": 12777714688, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 84230012928, + "reserved_bytes_peak": 84230012928, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.629258Z", + "world_size": 8 + }, + "allocated_bytes_current": 75893799936, + "allocated_bytes_current_delta": 32339738112, + "allocated_bytes_peak": 83802752512, + "allocated_bytes_peak_delta": 40242399232, + "before": { + "active_bytes_current": 43554061824, + "active_bytes_peak": 43560353280, + "allocated_bytes_current": 43554061824, + "allocated_bytes_peak": 43560353280, + "device": 6, + "inactive_split_bytes_current": 1255784960, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:18:24.522493Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_current_delta": 5450940928, + "inactive_split_bytes_peak": 12777714688, + "inactive_split_bytes_peak_delta": 11165453312, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 84230012928, + "reserved_bytes_current_delta": 39405486080, + "reserved_bytes_peak": 84230012928, + "reserved_bytes_peak_delta": 39405486080, + "segment_count_current": 2137, + "segment_count_current_delta": 1247, + "segment_count_peak": 2137, + "segment_count_peak_delta": 1247, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.765468Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4be930709792d42816c73488ebcd32eb9aa9b7eb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191851629699Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e62b31d3ae9b21769af22ae8ff5e75be204c422a10985edd96cc86387b6a2e8a +size 7019932 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b7596120933ddb74f9e22007f89eccfb2d3af49e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75893799936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114556165632, + "active_bytes_peak_delta": 30753413120, + "after": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_peak": 12777714688, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:52.883742Z", + "world_size": 8 + }, + "allocated_bytes_current": 75893799936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114556165632, + "allocated_bytes_peak_delta": 30753413120, + "before": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 83802752512, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 83802752512, + "device": 6, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_peak": 12777714688, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 84230012928, + "reserved_bytes_peak": 84230012928, + "segment_count_current": 2137, + "segment_count_peak": 2137, + "timestamp_utc": "2026-08-26T19:18:51.785007Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12777714688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 114997329920, + "reserved_bytes_current_delta": 30767316992, + "reserved_bytes_peak": 114997329920, + "reserved_bytes_peak_delta": 30767316992, + "segment_count_current": 2774, + "segment_count_current_delta": 637, + "segment_count_peak": 2774, + "segment_count_peak_delta": 637, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.017435Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..572134a31a3183c1e4b48366c76a2ecc940dfe50 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191852884188Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ca3e1f913ed657fdafbf79a6a3d6fdb7f5e4782b4f3f31e47b76c97472cca4 +size 6875449 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21f57f3d8d5c4589928f9bfde20eb00644eae8d2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75893799936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114556165632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6708823040, + "inactive_split_bytes_peak": 12777714688, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:54.042070Z", + "world_size": 8 + }, + "allocated_bytes_current": 75893799936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114556165632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6706725888, + "inactive_split_bytes_peak": 12777714688, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:53.034090Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6708823040, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12777714688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 114997329920, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114997329920, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2774, + "segment_count_current_delta": 0, + "segment_count_peak": 2774, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.174478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0045acdcbf52b8a5f76117e89083ea4670ae3e02 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191854042519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f609beea11b9795ea10540f5922d0b427b66fd721a5a46e9b5af578f930f13 +size 6845717 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70d1b58340c294d008471ea88816331e05868e1b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910843392, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114556165632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75910843392, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75910843392, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_peak": 16363299840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121074876416, + "reserved_bytes_peak": 121074876416, + "segment_count_current": 3058, + "segment_count_peak": 3058, + "timestamp_utc": "2026-08-26T19:18:57.301252Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910843392, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114556165632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75893799936, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75893799936, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6708823040, + "inactive_split_bytes_peak": 12777714688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 114997329920, + "reserved_bytes_peak": 114997329920, + "segment_count_current": 2774, + "segment_count_peak": 2774, + "timestamp_utc": "2026-08-26T19:18:54.190092Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16363299840, + "inactive_split_bytes_peak_delta": 3585585152, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121074876416, + "reserved_bytes_current_delta": 6077546496, + "reserved_bytes_peak": 121074876416, + "reserved_bytes_peak_delta": 6077546496, + "segment_count_current": 3058, + "segment_count_current_delta": 284, + "segment_count_peak": 3058, + "segment_count_peak_delta": 284, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.443898Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea80d94a2f7606b92652a736cc4d1644061d869a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191857301739Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f46da35f5770806b60f8841b79c03601df77062e6e0591803e0171f56d018f2 +size 7494801 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f6bd10dd16e3d035217f3a52dcbbd531fbb7426 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114556165632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75910843392, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75910843392, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_peak": 16363299840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121074876416, + "reserved_bytes_peak": 121074876416, + "segment_count_current": 3058, + "segment_count_peak": 3058, + "timestamp_utc": "2026-08-26T19:18:57.833053Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114556165632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75910843392, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75910843392, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_peak": 16363299840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121074876416, + "reserved_bytes_peak": 121074876416, + "segment_count_current": 3058, + "segment_count_peak": 3058, + "timestamp_utc": "2026-08-26T19:18:57.461562Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16363299840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121074876416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121074876416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3058, + "segment_count_current_delta": 0, + "segment_count_peak": 3058, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.854026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6695ce68abb1a0a4f5d77f242b1b91472a80468c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191857833482Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96cda55a34c2b9d54f1d8f325f12071e2e3c60ec05f329d868ac7c40f2ce9f69 +size 894339 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.json new file mode 100644 index 0000000000000000000000000000000000000000..758b82fd9a81384f48aa5d2d01890191d50dc667 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114556165632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75910843392, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75910843392, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_peak": 16363299840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121074876416, + "reserved_bytes_peak": 121074876416, + "segment_count_current": 3058, + "segment_count_peak": 3058, + "timestamp_utc": "2026-08-26T19:18:58.215987Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114556165632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75910843392, + "active_bytes_peak": 114556165632, + "allocated_bytes_current": 75910843392, + "allocated_bytes_peak": 114556165632, + "device": 6, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_peak": 16363299840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121074876416, + "reserved_bytes_peak": 121074876416, + "segment_count_current": 3058, + "segment_count_peak": 3058, + "timestamp_utc": "2026-08-26T19:18:57.855664Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6809220096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16363299840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121074876416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121074876416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3058, + "segment_count_current_delta": 0, + "segment_count_peak": 3058, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.235031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47f232b302758a8a78666127d3d6bd40de7e2532 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191858216378Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a53e44c305f60ad816f49a921d0df76997cc3d1aa4cc003cfe342b4d61114460 +size 894339 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3b50b024a28c5a3352e618b46580d91d0967b3e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 32342596096, + "active_bytes_peak": 83806146560, + "active_bytes_peak_delta": 40245660672, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 7, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.629162Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 32342596096, + "allocated_bytes_peak": 83806146560, + "allocated_bytes_peak_delta": 40245660672, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 7, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:18:24.522263Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_current_delta": 5288699392, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 11017812480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 84250984448, + "reserved_bytes_current_delta": 39415971840, + "reserved_bytes_peak": 84250984448, + "reserved_bytes_peak_delta": 39415971840, + "segment_count_current": 2126, + "segment_count_current_delta": 1243, + "segment_count_peak": 2126, + "segment_count_peak_delta": 1243, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.pickle", + "timestamp_utc": "2026-08-26T19:18:51.763945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bcd578d2731990cccebb884981fe066f78449ba --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191851629659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfa87d41d186c320c9545094d774c368c832ecbd8edf748a90763c7ce534c1a9 +size 7002568 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ae979c3478aa604b461127cb65f312374dfa994 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 30782408192, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:52.883573Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 30782408192, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 83806146560, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 83806146560, + "device": 7, + "inactive_split_bytes_current": 6544351744, + "inactive_split_bytes_peak": 12630073856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 84250984448, + "reserved_bytes_peak": 84250984448, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:18:51.784158Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 30788288512, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 30788288512, + "segment_count_current": 2751, + "segment_count_current_delta": 625, + "segment_count_peak": 2751, + "segment_count_peak_delta": 625, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.pickle", + "timestamp_utc": "2026-08-26T19:18:53.015732Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..359ed19753c1c2bdb5892d21406224ad306f3719 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191852883994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d6f563cbdab5fb072a58d3fccdfa09b80bf19da65d9f7ca88e86bad8aec102f +size 6610723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94970c15cba568774cec1096033f821b9622ee88 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75896790528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.041950Z", + "world_size": 8 + }, + "allocated_bytes_current": 75896790528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:53.032037Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12630073856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115039272960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115039272960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2751, + "segment_count_current_delta": 0, + "segment_count_peak": 2751, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.pickle", + "timestamp_utc": "2026-08-26T19:18:54.172688Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e9bc1bb8e1a8cd27e810ac16ac137f5ae02933c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191854042375Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a8eded7598e5795307d7aac7dc6712ac76eb0f95ddfeea051be8cc41752f28 +size 6580515 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6bef1054d60582e8a0002068cd2f18c55db3abe1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.301140Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75896790528, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75896790528, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6546448896, + "inactive_split_bytes_peak": 12630073856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115039272960, + "reserved_bytes_peak": 115039272960, + "segment_count_current": 2751, + "segment_count_peak": 2751, + "timestamp_utc": "2026-08-26T19:18:54.189612Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 100397056, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 3570851840, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 5876219904, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 5876219904, + "segment_count_current": 3040, + "segment_count_current_delta": 289, + "segment_count_peak": 3040, + "segment_count_peak_delta": 289, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.443864Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c27d868c8ef2edf4967c81581e22667d692a4dbf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191857301725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:271a2a36512797217820c8efb0a1f83db3691ab5e1cd64dd65a4f22c73a062ae +size 7477265 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf65fd0a4a1958809090ebc8c52da99038183c3a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.832960Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.461644Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.pickle", + "timestamp_utc": "2026-08-26T19:18:57.853706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f29a5e14411a76523c427422f7ddfc1a7a5663b0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191857833403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4902766b1392b94f47983d2dc4f7fd0bd0671af79ee9445c40c061be1a2f5871 +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd37a8bb21b0f7b4b79a8d9611ce51904a9919d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75913833984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 114588554752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:58.215879Z", + "world_size": 8 + }, + "allocated_bytes_current": 75913833984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 114588554752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75913833984, + "active_bytes_peak": 114588554752, + "allocated_bytes_current": 75913833984, + "allocated_bytes_peak": 114588554752, + "device": 7, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_peak": 16200925696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120915492864, + "reserved_bytes_peak": 120915492864, + "segment_count_current": 3040, + "segment_count_peak": 3040, + "timestamp_utc": "2026-08-26T19:18:57.855295Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6646845952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16200925696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120915492864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120915492864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3040, + "segment_count_current_delta": 0, + "segment_count_peak": 3040, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.pickle", + "timestamp_utc": "2026-08-26T19:18:58.234618Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..884aac013301a0bf878db9b8be9149aafc4111b2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191858216275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1205e65185d527a3ad30d1e6a832c690404dc256c3fd79e20d7087055e09fd17 +size 877258 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..e6fb03fe9446321010639d36b6480f7b33639b73 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..23882bb88e795a5bdaf6583bdca9e5ceb9960185 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,2875 @@ +[19:17:54] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:17:55.618000 645869 torch/distributed/run.py:766] +W0826 19:17:55.618000 645869 torch/distributed/run.py:766] ***************************************** +W0826 19:17:55.618000 645869 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:17:55.618000 645869 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to 0 is connected to 0[Gloo] Rank is connected to [Gloo] Rank 0 peer ranks. 00 peer ranks. 0Expected number of connected peer ranks is : is connected to peer ranks. Expected number of connected peer ranks is : is connected to 00Expected number of connected peer ranks is : 00 + peer ranks. 0 + peer ranks. Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 19:18:13.893232325 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:18:13.893276018 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:18:13.893339178 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:18:13.893374188 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:18:13.893401029 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:18:13.893419262 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:18:13.893543115 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:18:14.049664002 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.794 seconds +time to initialize megatron (seconds): 6.911 +[after megatron is initialized] datetime: 2026-08-26 19:18:19 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.k_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.k_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.linear_proj.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.k_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.k_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:18:24 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:18:24 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4578.06, 4587.90) + train/valid/test-data-iterators-setup ..........: (71.51, 87.59) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:18:24 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:18:51] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 27109.1 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095286E+01 | loss scale: 1.0 | grad norm: 54.365 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72366.705078125 | max allocated: 79925.2666015625 | reserved: 80330.0 | max reserved: 80330.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:18:52] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1254.7 | throughput per GPU (TFLOP/s/GPU): 71.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095473E+01 | loss scale: 1.0 | grad norm: 56.507 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:18:54] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1158.3 | throughput per GPU (TFLOP/s/GPU): 76.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008739E+01 | loss scale: 1.0 | grad norm: 180.689 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:18:57] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 3259.4 | throughput per GPU (TFLOP/s/GPU): 27.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603769E+00 | loss scale: 1.0 | grad norm: 8.739 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:18:57] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 531.8 | throughput per GPU (TFLOP/s/GPU): 167.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307957E+00 | loss scale: 1.0 | grad norm: 8.337 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | + [2026-08-26 19:18:58] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 382.9 | throughput per GPU (TFLOP/s/GPU): 232.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.126928E+00 | loss scale: 1.0 | grad norm: 2.871 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34560 | min safe capacity: 1.0547 | +[after training is done] datetime: 2026-08-26 19:18:58 +[rank0]:[W826 19:18:59.520161893 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:19:00.738313153 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:19:00.738388652 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:19:00.740403411 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:19:00.741491621 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:19:00.742759471 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:19:01.090793466 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:19:01.273549991 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:19:04.311000 645869 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:19:04.311000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645960 closing signal SIGTERM +W0826 19:19:04.312000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645961 closing signal SIGTERM +W0826 19:19:04.312000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645962 closing signal SIGTERM +W0826 19:19:04.314000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645963 closing signal SIGTERM +W0826 19:19:04.315000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645964 closing signal SIGTERM +W0826 19:19:04.316000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645965 closing signal SIGTERM +W0826 19:19:04.319000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645966 closing signal SIGTERM +W0826 19:19:04.320000 645869 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 645967 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 645869 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c1327552a08f0c2371d62f4c9de31d31c991ef18 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2663 +1, 2663 +2, 2663 +3, 2663 +4, 2663 +5, 2663 +6, 2663 +7, 2663 +0, 41187 +1, 41187 +2, 41187 +3, 41187 +4, 41187 +5, 41187 +6, 41187 +7, 41187 +0, 47183 +1, 47269 +2, 47195 +3, 47157 +4, 47191 +5, 47195 +6, 47187 +7, 47197 +0, 48475 +1, 48477 +2, 48487 +3, 48449 +4, 48473 +5, 48487 +6, 48477 +7, 48487 +0, 48645 +1, 48647 +2, 48657 +3, 48619 +4, 48643 +5, 48657 +6, 48647 +7, 48657 +0, 48779 +1, 48781 +2, 48787 +3, 48753 +4, 48777 +5, 48787 +6, 48781 +7, 48789 +0, 49583 +1, 49603 +2, 49577 +3, 49577 +4, 49581 +5, 49577 +6, 49563 +7, 49583 +0, 49583 +1, 49603 +2, 49577 +3, 49577 +4, 49581 +5, 49577 +6, 49563 +7, 49583 +0, 84253 +1, 84255 +2, 84175 +3, 84245 +4, 84253 +5, 84195 +6, 84219 +7, 84219 +0, 84899 +1, 84901 +2, 84821 +3, 84891 +4, 84899 +5, 84841 +6, 84865 +7, 84865 +0, 84899 +1, 84901 +2, 84821 +3, 84891 +4, 84899 +5, 84841 +6, 84865 +7, 84865 +0, 84899 +1, 84901 +2, 84821 +3, 84891 +4, 84899 +5, 84841 +6, 84865 +7, 84865 +0, 85299 +1, 85301 +2, 85221 +3, 85291 +4, 85299 +5, 85243 +6, 85265 +7, 85265 +0, 85947 +1, 85949 +2, 85869 +3, 85939 +4, 85947 +5, 85891 +6, 85913 +7, 85913 +0, 86519 +1, 86541 +2, 86525 +3, 86543 +4, 86583 +5, 86511 +6, 86513 +7, 86499 +0, 116103 +1, 116085 +2, 116099 +3, 116109 +4, 116087 +5, 116131 +6, 116115 +7, 116123 +0, 116103 +1, 116085 +2, 116099 +3, 116109 +4, 116089 +5, 116131 +6, 116115 +7, 116123 +0, 116103 +1, 116085 +2, 116099 +3, 116109 +4, 116089 +5, 116131 +6, 116115 +7, 116123 +0, 113099 +1, 113081 +2, 113095 +3, 113105 +4, 113085 +5, 113127 +6, 113111 +7, 113119 +0, 111031 +1, 110751 +2, 111091 +3, 111683 +4, 111935 +5, 111705 +6, 112077 +7, 111109 +0, 4 +1, 4 +2, 4 +3, 4 +4, 110371 +5, 4 +6, 4 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da6ec449b8aab859763f5aca7e1d361ba46f0208 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75982586368, + "active_bytes_current_delta": 32430357504, + "active_bytes_peak": 81922434560, + "active_bytes_peak_delta": 38363914240, + "after": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 81922434560, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 81922434560, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 13203781120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 82353061888, + "reserved_bytes_peak": 82353061888, + "segment_count_current": 2051, + "segment_count_peak": 2051, + "timestamp_utc": "2026-08-26T19:20:07.567053Z", + "world_size": 8 + }, + "allocated_bytes_current": 75982586368, + "allocated_bytes_current_delta": 32430357504, + "allocated_bytes_peak": 81922434560, + "allocated_bytes_peak_delta": 38363914240, + "before": { + "active_bytes_current": 43552228864, + "active_bytes_peak": 43558520320, + "allocated_bytes_current": 43552228864, + "allocated_bytes_peak": 43558520320, + "device": 0, + "inactive_split_bytes_current": 1255520768, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T19:19:39.070382Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_current_delta": 3017802752, + "inactive_split_bytes_peak": 13203781120, + "inactive_split_bytes_peak_delta": 11591495168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 82353061888, + "reserved_bytes_current_delta": 37530632192, + "reserved_bytes_peak": 82353061888, + "reserved_bytes_peak_delta": 37530632192, + "segment_count_current": 2051, + "segment_count_current_delta": 1166, + "segment_count_peak": 2051, + "segment_count_peak_delta": 1166, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.702083Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e0d8c3b305358dacd12fd5813443671484627de --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192007567516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16eb7992dfc079809185be83dc1ebf51206b8b413e7ee6d0808aedaa1c1eb51 +size 6893411 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b511ab8971eefc8a0c182fcdd88fbd39752adeac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75982586368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112708617728, + "active_bytes_peak_delta": 30786183168, + "after": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 13203781120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:09.539208Z", + "world_size": 8 + }, + "allocated_bytes_current": 75982586368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112708617728, + "allocated_bytes_peak_delta": 30786183168, + "before": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 81922434560, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 81922434560, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 13203781120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 82353061888, + "reserved_bytes_peak": 82353061888, + "segment_count_current": 2051, + "segment_count_peak": 2051, + "timestamp_utc": "2026-08-26T19:20:07.722639Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13203781120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_current_delta": 30805065728, + "reserved_bytes_peak": 113158127616, + "reserved_bytes_peak_delta": 30805065728, + "segment_count_current": 2736, + "segment_count_current_delta": 685, + "segment_count_peak": 2736, + "segment_count_peak_delta": 685, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.671801Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9bfb21ea0ef4dc3985ff22d2d41a4783ac09e0e1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192009539625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:224c47566f870e848173c91586bfb66d38971ca1f8e689edb6ecec7da53debec +size 6866269 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c53fbfe81f8c7d0801b9167d08deb49bce8e931 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75982586368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112708617728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 14805907968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:10.713448Z", + "world_size": 8 + }, + "allocated_bytes_current": 75982586368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112708617728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 13203781120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:09.687358Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14805907968, + "inactive_split_bytes_peak_delta": 1602126848, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113158127616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.845383Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d87c143a6909d75eae49f5ebba1f319329cb25a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192010713884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78ac8125baf8979716288ef160af1fc97566988d44027817cfe1aa6657a8b4d7 +size 7084902 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..945598632138bc6d4c119a333ed22fbd2baa913e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75982586368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112708617728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 15625829888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:11.781869Z", + "world_size": 8 + }, + "allocated_bytes_current": 75982586368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112708617728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 14805907968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:10.861022Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15625829888, + "inactive_split_bytes_peak_delta": 819921920, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113158127616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.912422Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b92bd51646596e99433a408465a9401a15e5c06 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192011782317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7f778161ebf696dbf08cb8221a9f7967d689452a5895b816cefe479a5e423bf +size 7083932 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b764cccccfcc1aef4f3a1417f3b9a2c3440f5613 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75982586368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112708617728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 15791976960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:12.835351Z", + "world_size": 8 + }, + "allocated_bytes_current": 75982586368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112708617728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75982586368, + "active_bytes_peak": 112708617728, + "allocated_bytes_current": 75982586368, + "allocated_bytes_peak": 112708617728, + "device": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_peak": 15625829888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_peak": 113158127616, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:11.927496Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4273323520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15791976960, + "inactive_split_bytes_peak_delta": 166147072, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113158127616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113158127616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.962972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9735ed51b7f3d58766e299214e7f8fb4bdee3490 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192012835789Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b2b20be8c2607dc9c5816f7e8166dccc009e259b09441cbecf59b229d81ff0 +size 7083792 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f756439b1d05965b17371b570a3675a8916e378 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76001884160, + "active_bytes_current_delta": 32447912448, + "active_bytes_peak": 81913597440, + "active_bytes_peak_delta": 38353334272, + "after": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 81913597440, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 81913597440, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 13439879168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 82355159040, + "reserved_bytes_peak": 82355159040, + "segment_count_current": 2060, + "segment_count_peak": 2060, + "timestamp_utc": "2026-08-26T19:20:07.566971Z", + "world_size": 8 + }, + "allocated_bytes_current": 76001884160, + "allocated_bytes_current_delta": 32447912448, + "allocated_bytes_peak": 81913597440, + "allocated_bytes_peak_delta": 38353334272, + "before": { + "active_bytes_current": 43553971712, + "active_bytes_peak": 43560263168, + "allocated_bytes_current": 43553971712, + "allocated_bytes_peak": 43560263168, + "device": 1, + "inactive_split_bytes_current": 1255875072, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:19:39.070418Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_current_delta": 2874418688, + "inactive_split_bytes_peak": 13439879168, + "inactive_split_bytes_peak_delta": 11827593216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 82355159040, + "reserved_bytes_current_delta": 37530632192, + "reserved_bytes_peak": 82355159040, + "reserved_bytes_peak_delta": 37530632192, + "segment_count_current": 2060, + "segment_count_current_delta": 1170, + "segment_count_peak": 2060, + "segment_count_peak_delta": 1170, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.701155Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a767894cc048a855338acce528daaef4c1891c48 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192007567499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1f786da35333b272eb326a70f376c047bf7bef61623b0e04236e56cb3a4f522 +size 7608088 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d14da0b3d048dd6effc8399aa8330cc4308211d3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76001884160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112683619840, + "active_bytes_peak_delta": 30770022400, + "after": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 13439879168, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:09.539195Z", + "world_size": 8 + }, + "allocated_bytes_current": 76001884160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112683619840, + "allocated_bytes_peak_delta": 30770022400, + "before": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 81913597440, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 81913597440, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 13439879168, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 82355159040, + "reserved_bytes_peak": 82355159040, + "segment_count_current": 2060, + "segment_count_peak": 2060, + "timestamp_utc": "2026-08-26T19:20:07.721226Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13439879168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113139253248, + "reserved_bytes_current_delta": 30784094208, + "reserved_bytes_peak": 113139253248, + "reserved_bytes_peak_delta": 30784094208, + "segment_count_current": 2754, + "segment_count_current_delta": 694, + "segment_count_peak": 2754, + "segment_count_peak_delta": 694, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.669663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2fba0e44fe9a3993633ebe7bc2d01b3398fa61d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192009539620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a53c9f935fa83598388e32bf5a8010f3309f41474ea8065c45bb2bcd4da7bc +size 6682440 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56e0b864a75b1b35c1fe6b1443b152ceef2a7842 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76001884160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112683619840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 14852192256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:10.713442Z", + "world_size": 8 + }, + "allocated_bytes_current": 76001884160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112683619840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 13439879168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:09.685254Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14852192256, + "inactive_split_bytes_peak_delta": 1412313088, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113139253248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113139253248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.842018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d7b92f3411e2c274cc666a245983071b7713790 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192010713885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9886b6b7bb58ee9215de2ecdf7eb8dd8c4f43dc16725e0a77a6a4857e700112d +size 6643428 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21b87622c438209da7ebda98b2bc3258d9880ce6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76001884160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112683619840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 15720254464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:11.781856Z", + "world_size": 8 + }, + "allocated_bytes_current": 76001884160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112683619840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 14852192256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:10.856902Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15720254464, + "inactive_split_bytes_peak_delta": 868062208, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113139253248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113139253248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.912323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d041b42d72eb1e2c90ff7818c7fd91f91d26f56 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192011782313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87e4686d457e29870f0c61602030f2a780483607cbb484a85b78e2592a57a366 +size 6642364 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.json new file mode 100644 index 0000000000000000000000000000000000000000..309d2d31fa49e67446049988fa29d5c06a8f2350 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76001884160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112683619840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 15720254464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:12.835345Z", + "world_size": 8 + }, + "allocated_bytes_current": 76001884160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112683619840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76001884160, + "active_bytes_peak": 112683619840, + "allocated_bytes_current": 76001884160, + "allocated_bytes_peak": 112683619840, + "device": 1, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_peak": 15720254464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113139253248, + "reserved_bytes_peak": 113139253248, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:11.927618Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4130293760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15720254464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113139253248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113139253248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.960903Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f527ff10a2b2bb41bc6b2da670f9ceec11a3e2f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192012835788Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4e2c183c8658b8cdf90a27fbc8ae6466223c71c85a3769c75d19ae7a01280a +size 6641954 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c9d30f26d55543c8ee01a749dd6da8a3941522e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75988771840, + "active_bytes_current_delta": 32434601984, + "active_bytes_peak": 81853082112, + "active_bytes_peak_delta": 38292620800, + "after": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 81853082112, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 81853082112, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 12983876608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 82271272960, + "reserved_bytes_peak": 82271272960, + "segment_count_current": 2045, + "segment_count_peak": 2045, + "timestamp_utc": "2026-08-26T19:20:07.566977Z", + "world_size": 8 + }, + "allocated_bytes_current": 75988771840, + "allocated_bytes_current_delta": 32434601984, + "allocated_bytes_peak": 81853082112, + "allocated_bytes_peak_delta": 38292620800, + "before": { + "active_bytes_current": 43554169856, + "active_bytes_peak": 43560461312, + "allocated_bytes_current": 43554169856, + "allocated_bytes_peak": 43560461312, + "device": 2, + "inactive_split_bytes_current": 1255676928, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:19:39.070366Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_current_delta": 3084861440, + "inactive_split_bytes_peak": 12983876608, + "inactive_split_bytes_peak_delta": 11371590656, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 82271272960, + "reserved_bytes_current_delta": 37436260352, + "reserved_bytes_peak": 82271272960, + "reserved_bytes_peak_delta": 37436260352, + "segment_count_current": 2045, + "segment_count_current_delta": 1162, + "segment_count_peak": 2045, + "segment_count_peak_delta": 1162, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.700664Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..403273db24b80a88afa8df7a6fa41cc02d2a7c73 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192007567498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3bd829ece523d5ecac2e940cd27052e9db8a01b2815412ac2f30ce6c1d7a468 +size 6880356 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6cb72d67863a80b6369e7ef869585237a065ec12 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75988771840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112681652736, + "active_bytes_peak_delta": 30828570624, + "after": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 12983876608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:09.539206Z", + "world_size": 8 + }, + "allocated_bytes_current": 75988771840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112681652736, + "allocated_bytes_peak_delta": 30828570624, + "before": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 81853082112, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 81853082112, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 12983876608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 82271272960, + "reserved_bytes_peak": 82271272960, + "segment_count_current": 2045, + "segment_count_peak": 2045, + "timestamp_utc": "2026-08-26T19:20:07.721369Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12983876608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113153933312, + "reserved_bytes_current_delta": 30882660352, + "reserved_bytes_peak": 113153933312, + "reserved_bytes_peak_delta": 30882660352, + "segment_count_current": 2715, + "segment_count_current_delta": 670, + "segment_count_peak": 2715, + "segment_count_peak_delta": 670, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.669663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2cf32a0e8e00cec533eb19a69e27a4f7db8a88d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192009539623Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3b80200b89bdc61b437a3648bb2e1ce1d10ece221dc4c30cf2ede7f7426cb56 +size 6561195 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56eed8796f28ae047b885d5a9ccccea10137e13a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75988771840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112681652736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 15056037888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:10.713433Z", + "world_size": 8 + }, + "allocated_bytes_current": 75988771840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112681652736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 12983876608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:09.685341Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15056037888, + "inactive_split_bytes_peak_delta": 2072161280, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113153933312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113153933312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2715, + "segment_count_current_delta": 0, + "segment_count_peak": 2715, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.843877Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6105a855fb64f0e8e6d09fb0207fb44d50c2c825 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192010713884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a79e11eb80249de88231983cd4210ba692bf96c803ac56ac283340ac60fe9c2 +size 6523687 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ddd57b2d9ad8039bd201e1e0dad1e829b8ce3d7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75988771840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112681652736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 15416923136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:11.781873Z", + "world_size": 8 + }, + "allocated_bytes_current": 75988771840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112681652736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 15056037888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:10.859914Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15416923136, + "inactive_split_bytes_peak_delta": 360885248, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113153933312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113153933312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2715, + "segment_count_current_delta": 0, + "segment_count_peak": 2715, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.914221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4853a5eb0b4b7ac2b65d5e3bdb137533ce787708 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192011782348Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724932e5c988965219cdb6cf65f05b0aefc24edb3915aa76ff0942b3b35d3fbd +size 6522588 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34f27c61d73c9578e0d47f4863264c55d11cb535 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75988771840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112681652736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 15667916800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:12.835350Z", + "world_size": 8 + }, + "allocated_bytes_current": 75988771840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112681652736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75988771840, + "active_bytes_peak": 112681652736, + "allocated_bytes_current": 75988771840, + "allocated_bytes_peak": 112681652736, + "device": 2, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_peak": 15416923136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113153933312, + "reserved_bytes_peak": 113153933312, + "segment_count_current": 2715, + "segment_count_peak": 2715, + "timestamp_utc": "2026-08-26T19:20:11.930243Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4340538368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15667916800, + "inactive_split_bytes_peak_delta": 250993664, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113153933312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113153933312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2715, + "segment_count_current_delta": 0, + "segment_count_peak": 2715, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.962287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..985d78213822d2af7d34176b9a73aca1aed57127 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192012835795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0edac3c46337b5221561a8b8857cfc6ab880acdb0d75171e3e68b500675e2f +size 6522185 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ff357801ac9547fd5bccceb871fa0140c18b0ba --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76005010944, + "active_bytes_current_delta": 32447561728, + "active_bytes_peak": 81919214592, + "active_bytes_peak_delta": 38355473920, + "after": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 81919214592, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 81919214592, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 13678680576, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 82344673280, + "reserved_bytes_peak": 82344673280, + "segment_count_current": 2065, + "segment_count_peak": 2065, + "timestamp_utc": "2026-08-26T19:20:07.567101Z", + "world_size": 8 + }, + "allocated_bytes_current": 76005010944, + "allocated_bytes_current_delta": 32447561728, + "allocated_bytes_peak": 81919214592, + "allocated_bytes_peak_delta": 38355473920, + "before": { + "active_bytes_current": 43557449216, + "active_bytes_peak": 43563740672, + "allocated_bytes_current": 43557449216, + "allocated_bytes_peak": 43563740672, + "device": 3, + "inactive_split_bytes_current": 1223037440, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T19:19:39.070298Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_current_delta": 2748940288, + "inactive_split_bytes_peak": 13678680576, + "inactive_split_bytes_peak_delta": 12066394624, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 82344673280, + "reserved_bytes_current_delta": 37549506560, + "reserved_bytes_peak": 82344673280, + "reserved_bytes_peak_delta": 37549506560, + "segment_count_current": 2065, + "segment_count_current_delta": 1178, + "segment_count_peak": 2065, + "segment_count_peak_delta": 1178, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.703477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d053bea17d8d9358da2022663e5e019a08deddfa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192007567619Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9d568cee47abff9c3cdb08250293ff63e90a5df80841a43869ba7d4f0bf8d0 +size 6898986 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80ff44c4b5e1c35c11ab90be4056726eadc111e7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76005010944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112712996352, + "active_bytes_peak_delta": 30793781760, + "after": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 13678680576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:09.539306Z", + "world_size": 8 + }, + "allocated_bytes_current": 76005010944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112712996352, + "allocated_bytes_peak_delta": 30793781760, + "before": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 81919214592, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 81919214592, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 13678680576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 82344673280, + "reserved_bytes_peak": 82344673280, + "segment_count_current": 2065, + "segment_count_peak": 2065, + "timestamp_utc": "2026-08-26T19:20:07.723787Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13678680576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113164419072, + "reserved_bytes_current_delta": 30819745792, + "reserved_bytes_peak": 113164419072, + "reserved_bytes_peak_delta": 30819745792, + "segment_count_current": 2754, + "segment_count_current_delta": 689, + "segment_count_peak": 2754, + "segment_count_peak_delta": 689, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.674792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ce26571cbd00be75dfe5e8e26b5e1ef49c25eed --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192009539750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3eaa4d1ed3900fe46587819edd5f2d6c5d00a0140df9debb42be61823cc7e2e +size 6825358 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ca300a914527997a19c09d81e8ea04b05588c8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76005010944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112712996352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 15138919936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:10.713606Z", + "world_size": 8 + }, + "allocated_bytes_current": 76005010944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112712996352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 13678680576, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:09.690642Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15138919936, + "inactive_split_bytes_peak_delta": 1460239360, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113164419072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113164419072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.846946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c27a92bbfa1436258b5cd4fcf4dc1e50fa7f4ab9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192010714061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c958b5aefebe4f8f1837600f5678349608e67b450ae7997d7169c07f039a5e +size 7089503 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0341aba9ad3468e4d26fef0446b728643e2df7f4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76005010944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112712996352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 15138919936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:11.781949Z", + "world_size": 8 + }, + "allocated_bytes_current": 76005010944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112712996352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 15138919936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:10.863196Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15138919936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113164419072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113164419072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.916059Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1bf821d8c2960d0d366011fcfcd92610efc5dd0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192011782460Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a8225ef789505522c0fbbae79166b9b626ceb83a23380402eac96564faa548 +size 7088293 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79504ee91ecf5b18194aa474a7466a3ae05abc71 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76005010944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112712996352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 15138919936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:12.835463Z", + "world_size": 8 + }, + "allocated_bytes_current": 76005010944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112712996352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76005010944, + "active_bytes_peak": 112712996352, + "allocated_bytes_current": 76005010944, + "allocated_bytes_peak": 112712996352, + "device": 3, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_peak": 15138919936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113164419072, + "reserved_bytes_peak": 113164419072, + "segment_count_current": 2754, + "segment_count_peak": 2754, + "timestamp_utc": "2026-08-26T19:20:11.932004Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3971977728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15138919936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113164419072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113164419072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2754, + "segment_count_current_delta": 0, + "segment_count_peak": 2754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.966596Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a66ef9d35cd6b4378ffc18effa26f65962839e92 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192012835926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3298a4c5240fa861f7dd49a12641ea9f309769020ebd8bd6c37299166cafa646 +size 7088111 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1c7c3e1365f90e1244906ff34d7ecfa9986cf7c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75989572096, + "active_bytes_current_delta": 32438267904, + "active_bytes_peak": 81926000128, + "active_bytes_peak_delta": 38368404480, + "after": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 81926000128, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 81926000128, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 13453206528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 82353061888, + "reserved_bytes_peak": 82353061888, + "segment_count_current": 2059, + "segment_count_peak": 2059, + "timestamp_utc": "2026-08-26T19:20:07.566942Z", + "world_size": 8 + }, + "allocated_bytes_current": 75989572096, + "allocated_bytes_current_delta": 32438267904, + "allocated_bytes_peak": 81926000128, + "allocated_bytes_peak_delta": 38368404480, + "before": { + "active_bytes_current": 43551304192, + "active_bytes_peak": 43557595648, + "allocated_bytes_current": 43551304192, + "allocated_bytes_peak": 43557595648, + "device": 4, + "inactive_split_bytes_current": 1254348288, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T19:19:39.070206Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_current_delta": 2888257536, + "inactive_split_bytes_peak": 13453206528, + "inactive_split_bytes_peak_delta": 11840920576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 82353061888, + "reserved_bytes_current_delta": 37532729344, + "reserved_bytes_peak": 82353061888, + "reserved_bytes_peak_delta": 37532729344, + "segment_count_current": 2059, + "segment_count_current_delta": 1170, + "segment_count_peak": 2059, + "segment_count_peak_delta": 1170, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.701215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..237f9f737ff422ace70e6d6cf6eb847b0e9a7a97 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192007567484Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dad4209da39c2b2c43a78ec87f4b9b4f90035659ea415916e49ecb8316af75d +size 6897432 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d13fa3ad17448b4a35a8190f4fce1c2ebc66e84 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75989572096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112664693248, + "active_bytes_peak_delta": 30738693120, + "after": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 13453206528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113141350400, + "reserved_bytes_peak": 113141350400, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-26T19:20:09.539225Z", + "world_size": 8 + }, + "allocated_bytes_current": 75989572096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112664693248, + "allocated_bytes_peak_delta": 30738693120, + "before": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 81926000128, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 81926000128, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 13453206528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 82353061888, + "reserved_bytes_peak": 82353061888, + "segment_count_current": 2059, + "segment_count_peak": 2059, + "timestamp_utc": "2026-08-26T19:20:07.721669Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13453206528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113141350400, + "reserved_bytes_current_delta": 30788288512, + "reserved_bytes_peak": 113141350400, + "reserved_bytes_peak_delta": 30788288512, + "segment_count_current": 2744, + "segment_count_current_delta": 685, + "segment_count_peak": 2744, + "segment_count_peak_delta": 685, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.670134Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe159ea2143a489604b0eca4e77758ef46d18b6e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192009539635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22b6905236d27b98824bdcdff38821b3a81e654dfc9bf87a43e8fce5b5fe3830 +size 7126139 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eae34f66dfd36dda8154eaa700f60dee2d990ebf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75989572096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112664693248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 14770942976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113143447552, + "reserved_bytes_peak": 113143447552, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-26T19:20:10.713453Z", + "world_size": 8 + }, + "allocated_bytes_current": 75989572096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112664693248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 13453206528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113141350400, + "reserved_bytes_peak": 113141350400, + "segment_count_current": 2744, + "segment_count_peak": 2744, + "timestamp_utc": "2026-08-26T19:20:09.685703Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14770942976, + "inactive_split_bytes_peak_delta": 1317736448, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113143447552, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 113143447552, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 2745, + "segment_count_current_delta": 1, + "segment_count_peak": 2745, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.840769Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56815ff9ac7194ef9caa4dc393ffab578cbef6b4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192010713875Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60a421b8eb281969da17b48db7c7bb5d066dc4d9f90ace0fa7d44be83f6f6847 +size 7088884 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..698184700eaab61632a4603db39638020a9f9fd7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75989572096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112664693248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 16003478528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113143447552, + "reserved_bytes_peak": 113143447552, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-26T19:20:11.781844Z", + "world_size": 8 + }, + "allocated_bytes_current": 75989572096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112664693248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 14770942976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113143447552, + "reserved_bytes_peak": 113143447552, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-26T19:20:10.856233Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16003478528, + "inactive_split_bytes_peak_delta": 1232535552, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113143447552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113143447552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2745, + "segment_count_current_delta": 0, + "segment_count_peak": 2745, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.907870Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f1490f6d50de62f17a918cb9d3cba978e35d896 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192011782269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3318b7baea05eeef05cdadcce458a2ab3073f0376cbcf820437a35e74a3d4fc +size 7086639 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.json new file mode 100644 index 0000000000000000000000000000000000000000..daddfad38dbe403957b1b5a1787e8ee659db476d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75989572096, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112664693248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 16003478528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113143447552, + "reserved_bytes_peak": 113143447552, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-26T19:20:12.835337Z", + "world_size": 8 + }, + "allocated_bytes_current": 75989572096, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112664693248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75989572096, + "active_bytes_peak": 112664693248, + "allocated_bytes_current": 75989572096, + "allocated_bytes_peak": 112664693248, + "device": 4, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_peak": 16003478528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113143447552, + "reserved_bytes_peak": 113143447552, + "segment_count_current": 2745, + "segment_count_peak": 2745, + "timestamp_utc": "2026-08-26T19:20:11.923167Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4142605824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16003478528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113143447552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113143447552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2745, + "segment_count_current_delta": 0, + "segment_count_peak": 2745, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.962536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd4f9a7798d9f276eb8d258687dfe627d2c89648 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192012835747Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:289b2327a55fb308b6153e05a2f8fcef37a7bcecccd65e74006b41e5b2349791 +size 7086461 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6848e0b7b12a23fa7f494d5d043fb9885f7b5705 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75986853888, + "active_bytes_current_delta": 32432684032, + "active_bytes_peak": 81869163008, + "active_bytes_peak_delta": 38308701696, + "after": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 81869163008, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 81869163008, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 13085677568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 82294341632, + "reserved_bytes_peak": 82294341632, + "segment_count_current": 2047, + "segment_count_peak": 2047, + "timestamp_utc": "2026-08-26T19:20:07.566948Z", + "world_size": 8 + }, + "allocated_bytes_current": 75986853888, + "allocated_bytes_current_delta": 32432684032, + "allocated_bytes_peak": 81869163008, + "allocated_bytes_peak_delta": 38308701696, + "before": { + "active_bytes_current": 43554169856, + "active_bytes_peak": 43560461312, + "allocated_bytes_current": 43554169856, + "allocated_bytes_peak": 43560461312, + "device": 5, + "inactive_split_bytes_current": 1255676928, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:19:39.070275Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_current_delta": 2965144576, + "inactive_split_bytes_peak": 13085677568, + "inactive_split_bytes_peak_delta": 11473391616, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 82294341632, + "reserved_bytes_current_delta": 37459329024, + "reserved_bytes_peak": 82294341632, + "reserved_bytes_peak_delta": 37459329024, + "segment_count_current": 2047, + "segment_count_current_delta": 1164, + "segment_count_peak": 2047, + "segment_count_peak_delta": 1164, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.700660Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef79b196acd2df23ebbb746e476c3c83ad80b5b7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192007567498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec5067b6751b36bbf40cd35982e0b008f4881ca3b557e00f33bf8142013b93c +size 6880245 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2031e0e2ce0574b86a86a90754e79faf7453e1f6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75986853888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112714079232, + "active_bytes_peak_delta": 30844916224, + "after": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 13085677568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:09.539197Z", + "world_size": 8 + }, + "allocated_bytes_current": 75986853888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112714079232, + "allocated_bytes_peak_delta": 30844916224, + "before": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 81869163008, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 81869163008, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 13085677568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 82294341632, + "reserved_bytes_peak": 82294341632, + "segment_count_current": 2047, + "segment_count_peak": 2047, + "timestamp_utc": "2026-08-26T19:20:07.721100Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13085677568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113187487744, + "reserved_bytes_current_delta": 30893146112, + "reserved_bytes_peak": 113187487744, + "reserved_bytes_peak_delta": 30893146112, + "segment_count_current": 2736, + "segment_count_current_delta": 689, + "segment_count_peak": 2736, + "segment_count_peak_delta": 689, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.676081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27c14a2d67228023bf94022ba2475c4a4d2f2bb5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192009539644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eb33c96972af8bdcc43a810c19cd03218d6223112a6a47bbd8653f98b318e98 +size 6544428 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4fc9b814a7292835963a398be6172bc121b84d2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75986853888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112714079232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 14601816064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:10.713448Z", + "world_size": 8 + }, + "allocated_bytes_current": 75986853888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112714079232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 13085677568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:09.691544Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14601816064, + "inactive_split_bytes_peak_delta": 1516138496, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113187487744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113187487744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.841837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b474aaedeac731036aa4abc6c8b6b202a8ecf58a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192010713878Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d2e4ea5311c1d71231c2aee96828ae6430bbdb11f8a1c0ecd7f1ad5ca56fa5 +size 6506050 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2858651125d287a319fc384f64f126c10aa75a3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75986853888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112714079232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 15858886656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:11.781818Z", + "world_size": 8 + }, + "allocated_bytes_current": 75986853888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112714079232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 14601816064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:10.856884Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15858886656, + "inactive_split_bytes_peak_delta": 1257070592, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113187487744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113187487744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.pickle", + "timestamp_utc": "2026-08-26T19:20:11.907807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c53154cf075c5b2624fa735d412481c3c965d996 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192011782250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6965754a669cd5a89bcb41cec3642e14f1f7ca101f214420434ef76c1d7b7416 +size 6504782 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c8778a6fb26c07a0195da2e87b4274a60e683f7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75986853888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112714079232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 16044076032, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:12.835326Z", + "world_size": 8 + }, + "allocated_bytes_current": 75986853888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112714079232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75986853888, + "active_bytes_peak": 112714079232, + "allocated_bytes_current": 75986853888, + "allocated_bytes_peak": 112714079232, + "device": 5, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_peak": 15858886656, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113187487744, + "reserved_bytes_peak": 113187487744, + "segment_count_current": 2736, + "segment_count_peak": 2736, + "timestamp_utc": "2026-08-26T19:20:11.923042Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4220821504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16044076032, + "inactive_split_bytes_peak_delta": 185189376, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113187487744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113187487744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2736, + "segment_count_current_delta": 0, + "segment_count_peak": 2736, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.pickle", + "timestamp_utc": "2026-08-26T19:20:12.959829Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b6f9216f45b02eb2d78ceb43e9ddb308389aea1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192012835743Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae78dd88ce0c01c8b208c68497a45f0489c10a5df31d38c13264fb606250dcfa +size 6524035 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..183904ee971802c3a0fad39e3857e68f2a5efce4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76000535552, + "active_bytes_current_delta": 32446498304, + "active_bytes_peak": 81902876672, + "active_bytes_peak_delta": 38342547968, + "after": { + "active_bytes_current": 76000535552, + "active_bytes_peak": 81902876672, + "allocated_bytes_current": 76000535552, + "allocated_bytes_peak": 81902876672, + "device": 6, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_peak": 13073498112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 82317410304, + "reserved_bytes_peak": 82317410304, + "segment_count_current": 2063, + "segment_count_peak": 2063, + "timestamp_utc": "2026-08-26T19:20:07.566954Z", + "world_size": 8 + }, + "allocated_bytes_current": 76000535552, + "allocated_bytes_current_delta": 32446498304, + "allocated_bytes_peak": 81902876672, + "allocated_bytes_peak_delta": 38342547968, + "before": { + "active_bytes_current": 43554037248, + "active_bytes_peak": 43560328704, + "allocated_bytes_current": 43554037248, + "allocated_bytes_peak": 43560328704, + "device": 6, + "inactive_split_bytes_current": 1255809536, + "inactive_split_bytes_peak": 1612285952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:19:39.070301Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_current_delta": 2835986944, + "inactive_split_bytes_peak": 13073498112, + "inactive_split_bytes_peak_delta": 11461212160, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 82317410304, + "reserved_bytes_current_delta": 37492883456, + "reserved_bytes_peak": 82317410304, + "reserved_bytes_peak_delta": 37492883456, + "segment_count_current": 2063, + "segment_count_current_delta": 1173, + "segment_count_peak": 2063, + "segment_count_peak_delta": 1173, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.pickle", + "timestamp_utc": "2026-08-26T19:20:07.701400Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e4d11cc5311df88b8b749b6b8e45fe890580fdb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192007567497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb3efc9e0cdf0cf6b2ac8cc58c63a71e5f5271ba870fba41b6af7bb63324b99 +size 6899446 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba74e7e95e860c20957d2b7add444dea28011698 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76000535552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112732114432, + "active_bytes_peak_delta": 30829237760, + "after": { + "active_bytes_current": 76000535552, + "active_bytes_peak": 112732114432, + "allocated_bytes_current": 76000535552, + "allocated_bytes_peak": 112732114432, + "device": 6, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_peak": 13127609344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113170710528, + "reserved_bytes_peak": 113170710528, + "segment_count_current": 2765, + "segment_count_peak": 2765, + "timestamp_utc": "2026-08-26T19:20:09.539176Z", + "world_size": 8 + }, + "allocated_bytes_current": 76000535552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112732114432, + "allocated_bytes_peak_delta": 30829237760, + "before": { + "active_bytes_current": 76000535552, + "active_bytes_peak": 81902876672, + "allocated_bytes_current": 76000535552, + "allocated_bytes_peak": 81902876672, + "device": 6, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_peak": 13073498112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 82317410304, + "reserved_bytes_peak": 82317410304, + "segment_count_current": 2063, + "segment_count_peak": 2063, + "timestamp_utc": "2026-08-26T19:20:07.721938Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13127609344, + "inactive_split_bytes_peak_delta": 54111232, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113170710528, + "reserved_bytes_current_delta": 30853300224, + "reserved_bytes_peak": 113170710528, + "reserved_bytes_peak_delta": 30853300224, + "segment_count_current": 2765, + "segment_count_current_delta": 702, + "segment_count_peak": 2765, + "segment_count_peak_delta": 702, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.pickle", + "timestamp_utc": "2026-08-26T19:20:09.672081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..988327b791e8d6c0a6f756c265fb6182ce6cd20c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192009539608Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f6bc3fe2a29f114df07b8297424bc9a8d9f7a1ce29f686a94aede80764fe8bc +size 6684436 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8730c5514ff73abb2c407d2cd98087ca5be29da1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 76000535552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112732114432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 76000535552, + "active_bytes_peak": 112732114432, + "allocated_bytes_current": 76000535552, + "allocated_bytes_peak": 112732114432, + "device": 6, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_peak": 14970116096, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113170710528, + "reserved_bytes_peak": 113170710528, + "segment_count_current": 2765, + "segment_count_peak": 2765, + "timestamp_utc": "2026-08-26T19:20:10.713478Z", + "world_size": 8 + }, + "allocated_bytes_current": 76000535552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112732114432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 76000535552, + "active_bytes_peak": 112732114432, + "allocated_bytes_current": 76000535552, + "allocated_bytes_peak": 112732114432, + "device": 6, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_peak": 13127609344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113170710528, + "reserved_bytes_peak": 113170710528, + "segment_count_current": 2765, + "segment_count_peak": 2765, + "timestamp_utc": "2026-08-26T19:20:09.688557Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4091796480, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14970116096, + "inactive_split_bytes_peak_delta": 1842506752, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113170710528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113170710528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2765, + "segment_count_current_delta": 0, + "segment_count_peak": 2765, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.pickle", + "timestamp_utc": "2026-08-26T19:20:10.841097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84c34038824cc5bc2344c83c5d65803072a260bf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192010713912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b7ebd5b8c70af466a1d702881f8b1bbbd6a5b9036079a4da9fbf32776f421b +size 6644616